From 932dcac7c6c73706afe8d2220e028f13f1a44bbb Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 4 Aug 2022 19:47:25 -0700 Subject: [PATCH 01/32] events and projects --- .../source-sentry/source_sentry/sentry.yaml | 62 +++++++++++++++++++ .../source-sentry/source_sentry/source.py | 47 +++----------- 2 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml new file mode 100644 index 0000000000000..4665376e25017 --- /dev/null +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml @@ -0,0 +1,62 @@ +definitions: + page_size: 50 + + schema_loader: + type: JsonSchema + file_path: "./source_sentry/schemas/{{ options.name }}.json" + selector: + type: RecordSelector + extractor: + type: JelloExtractor + transform: "_" + requester: + type: HttpRequester + name: "{{ options['name'] }}" + url_base: "https://{{ config.hostname }}/api/0/" + http_method: "GET" + authenticator: + type: "BearerAuthenticator" + token: "{{ config.auth_token }}" + retriever: + type: SimpleRetriever + name: "{{ options['name'] }}" + primary_key: "{{ options['primary_key'] }}" + +streams: + - type: DeclarativeStream + $options: + name: "events" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector)" + requester: + $ref: "*ref(definitions.requester)" + path: "projects/{{config.organization}}/{{config.project}}/events/" + request_options_provider: + request_parameters: + full: "true" + paginator: + type: NoPagination + - type: DeclarativeStream + $options: + name: "projects" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector)" + requester: + $ref: "*ref(definitions.requester)" + path: "projects/" + paginator: + type: NoPagination + +check: + type: CheckStream + stream_names: ["events"] diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/source.py b/airbyte-integrations/connectors/source-sentry/source_sentry/source.py index 398ec1274de40..5d68e9a6172dd 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/source.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/source.py @@ -2,43 +2,16 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from .streams import Events, Issues, ProjectDetail, Projects - - -# Source -class SourceSentry(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, Any]: - try: - projects_stream = Projects( - authenticator=TokenAuthenticator(token=config["auth_token"]), - hostname=config.get("hostname"), - ) - next(projects_stream.read_records(sync_mode=SyncMode.full_refresh)) - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - stream_args = { - "authenticator": TokenAuthenticator(token=config["auth_token"]), - "hostname": config.get("hostname"), - } - project_stream_args = { - **stream_args, - "organization": config["organization"], - "project": config["project"], - } - return [ - Events(**project_stream_args), - Issues(**project_stream_args), - ProjectDetail(**project_stream_args), - Projects(**stream_args), - ] +# Declarative Source +class SourceSentry(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "./source_sentry/sentry.yaml"}) From 0c0135b609dd91c4281445e19d24a3274ac26a77 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 4 Aug 2022 20:00:10 -0700 Subject: [PATCH 02/32] done minus pagination --- .../source-sentry/source_sentry/sentry.yaml | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml index 4665376e25017..827aa2558e98e 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml @@ -41,6 +41,25 @@ streams: full: "true" paginator: type: NoPagination + - type: DeclarativeStream + $options: + name: "issues" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector)" + requester: + $ref: "*ref(definitions.requester)" + path: "projects/{{config.organization}}/{{config.project}}/issues/" + request_options_provider: + request_parameters: + statsPeriod: "" + query: "" + paginator: + type: NoPagination - type: DeclarativeStream $options: name: "projects" @@ -56,7 +75,21 @@ streams: path: "projects/" paginator: type: NoPagination - + - type: DeclarativeStream + $options: + name: "project_detail" + primary_key: "id" + schema_loader: + $ref: "*ref(definitions.schema_loader)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector)" + requester: + $ref: "*ref(definitions.requester)" + path: "projects/{{config.organization}}/{{config.project}}/" + paginator: + type: NoPagination check: type: CheckStream stream_names: ["events"] From f473dee6707320e26c9c4e5ea4589d4ec87eed9f Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 4 Aug 2022 20:02:50 -0700 Subject: [PATCH 03/32] handle single records --- .../sources/declarative/extractors/record_selector.py | 2 ++ .../sources/declarative/extractors/test_record_selector.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py index 193f0e7576eba..c20bf4562ad95 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py @@ -35,6 +35,8 @@ def select_records( next_page_token: Optional[Mapping[str, Any]] = None, ) -> List[Record]: all_records = self._extractor.extract_records(response) + if not isinstance(all_records, list): + all_records = [all_records] if self._record_filter: return self._record_filter.filter_records( all_records, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py index 0367d7d34a18d..64115e53cff8a 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py @@ -29,6 +29,13 @@ {"data": [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}]}, [{"id": 1, "created_at": "06-06-21"}, {"id": 2, "created_at": "06-07-21"}], ), + ( + "test_read_single_record", + "_.data", + None, + {"data": {"id": 1, "created_at": "06-06-21"}}, + [{"id": 1, "created_at": "06-06-21"}], + ), ], ) def test_record_filter(test_name, transform_template, filter_template, body, expected_records): From 886c24ab9a1737be8642a99550ad425b4023154d Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Tue, 9 Aug 2022 16:43:45 -0700 Subject: [PATCH 04/32] pagination --- .../source-sentry/source_sentry/sentry.yaml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml index 827aa2558e98e..155826704b04a 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml @@ -1,6 +1,5 @@ definitions: page_size: 50 - schema_loader: type: JsonSchema file_path: "./source_sentry/schemas/{{ options.name }}.json" @@ -16,7 +15,20 @@ definitions: http_method: "GET" authenticator: type: "BearerAuthenticator" - token: "{{ config.auth_token }}" + api_token: "{{ config.auth_token }}" + paginator: + type: LimitPaginator + url_base: "*ref(definitions.requester.url_base)" + page_size: "*ref(definitions.page_size)" + limit_option: + inject_into: "request_parameter" + field_name: "" + page_token_option: + inject_into: "path" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ headers.link.next.cursor }}" + stop_condition: "{{ headers.link.results != true }}" retriever: type: SimpleRetriever name: "{{ options['name'] }}" @@ -40,7 +52,7 @@ streams: request_parameters: full: "true" paginator: - type: NoPagination + $ref: "*ref(definitions.paginator)" - type: DeclarativeStream $options: name: "issues" From e234a8d5602f2ce3b11c9a833a8404dedb5591d3 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Tue, 9 Aug 2022 16:47:00 -0700 Subject: [PATCH 05/32] bump min cdk version --- airbyte-integrations/connectors/source-sentry/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-sentry/setup.py b/airbyte-integrations/connectors/source-sentry/setup.py index 8b20427c76b9b..2ebf8d6a31cb2 100644 --- a/airbyte-integrations/connectors/source-sentry/setup.py +++ b/airbyte-integrations/connectors/source-sentry/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1.72", ] TEST_REQUIREMENTS = [ From 4bd72f199170c1e981245a19bbc17ad285fa9adb Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 10 Aug 2022 07:49:48 -0700 Subject: [PATCH 06/32] start on unit tests --- .../source-sentry/source_sentry/sentry.yaml | 2 +- .../source-sentry/unit_tests/test_source.py | 26 ----------------- .../source-sentry/unit_tests/test_streams.py | 28 +++++++++++++++---- 3 files changed, 24 insertions(+), 32 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-sentry/unit_tests/test_source.py diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml index 155826704b04a..c3c298e541c33 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml @@ -28,7 +28,7 @@ definitions: pagination_strategy: type: "CursorPagination" cursor_value: "{{ headers.link.next.cursor }}" - stop_condition: "{{ headers.link.results != true }}" + stop_condition: "{{ headers.link.next.results != 'true' }}" retriever: type: SimpleRetriever name: "{{ options['name'] }}" diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_source.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_source.py deleted file mode 100644 index 2d81d29cea0d0..0000000000000 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_source.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_sentry.source import SourceSentry -from source_sentry.streams import Projects - - -def test_check_connection(mocker): - source = SourceSentry() - logger_mock, config_mock = MagicMock(), MagicMock() - mocker.patch.object(Projects, "read_records", return_value=iter([{"id": "1", "name": "test"}])) - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceSentry() - config_mock = MagicMock() - config_mock["auth_token"] = "test-token" - config_mock["organization"] = "test-organization" - config_mock["project"] = "test-project" - streams = source.streams(config_mock) - expected_streams_number = 4 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py index cfd5ef76a10cd..443c403ba7dc6 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py @@ -2,9 +2,12 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import json from unittest.mock import MagicMock import pytest +import requests +from source_sentry.source import SourceSentry from source_sentry.streams import Events, Issues, ProjectDetail, Projects, SentryStreamPagination INIT_ARGS = {"hostname": "sentry.io", "organization": "test-org", "project": "test-project"} @@ -17,14 +20,29 @@ def patch_base_class(mocker): mocker.patch.object(SentryStreamPagination, "__abstractmethods__", set()) +def create_response(links): + response = requests.Response() + response_body = {"next": "https://airbyte.io/next_url"} + response._content = json.dumps(response_body).encode("utf-8") + response.headers = links + return response + + def test_next_page_token(patch_base_class): - stream = SentryStreamPagination(hostname="sentry.io") - resp = MagicMock() + source = SourceSentry() + config = {} + streams = source.streams(config) cursor = "next_page_num" - resp.links = {"next": {"results": "true", "cursor": cursor}} + + stream = [s for s in streams if s.name == "events"][0] + resp = create_response( + { + "link": f'; rel="next"; results="true"; cursor="{cursor}"' + } + ) inputs = {"response": resp} - expected_token = {"cursor": cursor} - assert stream.next_page_token(**inputs) == expected_token + expected_token = {"next_page_token": cursor} + assert stream.retriever.next_page_token(**inputs) == expected_token def test_next_page_token_is_none(patch_base_class): From e1c0ae02440fc57b86bdda4c16bfb41371d08269 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 10 Aug 2022 07:58:08 -0700 Subject: [PATCH 07/32] Update more unit tests --- .../source-sentry/unit_tests/test_streams.py | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py index 443c403ba7dc6..f31ffb1b50a9a 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py @@ -30,8 +30,7 @@ def create_response(links): def test_next_page_token(patch_base_class): source = SourceSentry() - config = {} - streams = source.streams(config) + streams = source.streams(INIT_ARGS) cursor = "next_page_num" stream = [s for s in streams if s.name == "events"][0] @@ -46,12 +45,18 @@ def test_next_page_token(patch_base_class): def test_next_page_token_is_none(patch_base_class): - stream = SentryStreamPagination(hostname="sentry.io") - resp = MagicMock() - resp.links = {"next": {"results": "false", "cursor": "no_next"}} + source = SourceSentry() + streams = source.streams(INIT_ARGS) + cursor = "next_page_num" + + stream = [s for s in streams if s.name == "events"][0] + resp = create_response( + { + "link": f'; rel="next"; results="false"; cursor="{cursor}"' + } + ) inputs = {"response": resp} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token + assert stream.retriever.next_page_token(**inputs) is None def next_page_token_inputs(): @@ -74,28 +79,35 @@ def test_next_page_token_raises(patch_base_class, response): stream.next_page_token(**inputs) +def get_stream(stream_name): + source = SourceSentry() + streams = source.streams(INIT_ARGS) + + return [s for s in streams if s.name == stream_name][0] + + def test_events_path(): - stream = Events(**INIT_ARGS) + stream = get_stream("events") expected = "projects/test-org/test-project/events/" - assert stream.path() == expected + assert stream.retriever.path() == expected def test_issues_path(): - stream = Issues(**INIT_ARGS) + stream = get_stream("issues") expected = "projects/test-org/test-project/issues/" - assert stream.path() == expected + assert stream.retriever.path() == expected def test_projects_path(): - stream = Projects(hostname="sentry.io") + stream = get_stream("projects") expected = "projects/" - assert stream.path() == expected + assert stream.retriever.path() == expected def test_project_detail_path(): - stream = ProjectDetail(**INIT_ARGS) + stream = get_stream("project_detail") expected = "projects/test-org/test-project/" - assert stream.path() == expected + assert stream.retriever.path() == expected def test_sentry_stream_pagination_request_params(patch_base_class): From 42af8170028ceef41a38a59d124a6f4ad824ee49 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 10 Aug 2022 08:47:02 -0700 Subject: [PATCH 08/32] Handle extracting no records from root --- .../declarative/extractors/record_selector.py | 2 ++ .../declarative/extractors/test_record_selector.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py index dd738a69015d9..f4a844daa9778 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py @@ -39,6 +39,8 @@ def select_records( next_page_token: Optional[Mapping[str, Any]] = None, ) -> List[Record]: all_records = self.extractor.extract_records(response) + if not all_records: + return [] # Some APIs don't wrap single records in a list if not isinstance(all_records, list): all_records = [all_records] diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py index fa2bbfdcd7ce5..f0b94c1cd6fd0 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py @@ -43,6 +43,20 @@ {"data": {"id": 1, "created_at": "06-06-21"}}, [{"id": 1, "created_at": "06-06-21"}], ), + ( + "test_no_record", + "_.data", + None, + {"data": []}, + [], + ), + ( + "test_no_record_from_root", + "_", + None, + [], + [], + ), ], ) def test_record_filter(test_name, transform_template, filter_template, body, expected_records): From ddae912daf7e0bb49b63d3a351a6ccf94d677042 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 10 Aug 2022 12:39:00 -0700 Subject: [PATCH 09/32] additionalProperties=true --- .../connectors/source-sentry/source_sentry/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/spec.json b/airbyte-integrations/connectors/source-sentry/source_sentry/spec.json index a1d9e35c0a35d..7820a6e6bcb30 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/spec.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/spec.json @@ -5,7 +5,7 @@ "title": "Sentry Spec", "type": "object", "required": ["auth_token", "organization", "project"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "auth_token": { "type": "string", From d1c0224d70adaf42d3b776c5e7c35f378e2208f6 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 10 Aug 2022 12:49:03 -0700 Subject: [PATCH 10/32] handle empty streams --- .../connectors/source-sentry/source_sentry/sentry.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml index c3c298e541c33..e9fb6d5eb9c0d 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml @@ -7,7 +7,7 @@ definitions: type: RecordSelector extractor: type: JelloExtractor - transform: "_" + transform: "_ or []" requester: type: HttpRequester name: "{{ options['name'] }}" From 144f816d01cde5224950038696d89aca525e9bd7 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 10 Aug 2022 12:59:35 -0700 Subject: [PATCH 11/32] skip backward compatibility tests --- .../connectors/source-sentry/acceptance-test-config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml index 6e27bef75cf46..2f9cf7f1a9ab6 100644 --- a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml @@ -2,6 +2,8 @@ connector_image: airbyte/source-sentry:dev tests: spec: - spec_path: "source_sentry/spec.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.1" connection: - config_path: "secrets/config.json" status: "succeed" From 0010cf3cef6f092c2904c439957eb6e10cb144fb Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 10 Aug 2022 13:36:28 -0700 Subject: [PATCH 12/32] check on project_detail --- .../connectors/source-sentry/source_sentry/sentry.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml index e9fb6d5eb9c0d..5f3f81a78b1db 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml @@ -104,4 +104,4 @@ streams: type: NoPagination check: type: CheckStream - stream_names: ["events"] + stream_names: ["project_detail"] From 793521e1950e3b7eaffc7ea679a969373d7da4b7 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 10 Aug 2022 13:54:49 -0700 Subject: [PATCH 13/32] remove unit tests --- .../source-sentry/unit_tests/test_streams.py | 140 ------------------ 1 file changed, 140 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py deleted file mode 100644 index f31ffb1b50a9a..0000000000000 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py +++ /dev/null @@ -1,140 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -import json -from unittest.mock import MagicMock - -import pytest -import requests -from source_sentry.source import SourceSentry -from source_sentry.streams import Events, Issues, ProjectDetail, Projects, SentryStreamPagination - -INIT_ARGS = {"hostname": "sentry.io", "organization": "test-org", "project": "test-project"} - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(SentryStreamPagination, "path", "test_endpoint") - mocker.patch.object(SentryStreamPagination, "__abstractmethods__", set()) - - -def create_response(links): - response = requests.Response() - response_body = {"next": "https://airbyte.io/next_url"} - response._content = json.dumps(response_body).encode("utf-8") - response.headers = links - return response - - -def test_next_page_token(patch_base_class): - source = SourceSentry() - streams = source.streams(INIT_ARGS) - cursor = "next_page_num" - - stream = [s for s in streams if s.name == "events"][0] - resp = create_response( - { - "link": f'; rel="next"; results="true"; cursor="{cursor}"' - } - ) - inputs = {"response": resp} - expected_token = {"next_page_token": cursor} - assert stream.retriever.next_page_token(**inputs) == expected_token - - -def test_next_page_token_is_none(patch_base_class): - source = SourceSentry() - streams = source.streams(INIT_ARGS) - cursor = "next_page_num" - - stream = [s for s in streams if s.name == "events"][0] - resp = create_response( - { - "link": f'; rel="next"; results="false"; cursor="{cursor}"' - } - ) - inputs = {"response": resp} - assert stream.retriever.next_page_token(**inputs) is None - - -def next_page_token_inputs(): - links_headers = [ - {}, - {"next": {}}, - ] - responses = [MagicMock() for _ in links_headers] - for mock, header in zip(responses, links_headers): - mock.links = header - - return responses - - -@pytest.mark.parametrize("response", next_page_token_inputs()) -def test_next_page_token_raises(patch_base_class, response): - stream = SentryStreamPagination(hostname="sentry.io") - inputs = {"response": response} - with pytest.raises(KeyError): - stream.next_page_token(**inputs) - - -def get_stream(stream_name): - source = SourceSentry() - streams = source.streams(INIT_ARGS) - - return [s for s in streams if s.name == stream_name][0] - - -def test_events_path(): - stream = get_stream("events") - expected = "projects/test-org/test-project/events/" - assert stream.retriever.path() == expected - - -def test_issues_path(): - stream = get_stream("issues") - expected = "projects/test-org/test-project/issues/" - assert stream.retriever.path() == expected - - -def test_projects_path(): - stream = get_stream("projects") - expected = "projects/" - assert stream.retriever.path() == expected - - -def test_project_detail_path(): - stream = get_stream("project_detail") - expected = "projects/test-org/test-project/" - assert stream.retriever.path() == expected - - -def test_sentry_stream_pagination_request_params(patch_base_class): - stream = SentryStreamPagination(hostname="sentry.io") - expected = {"cursor": "next-page"} - assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected - - -def test_events_request_params(): - stream = Events(**INIT_ARGS) - expected = {"cursor": "next-page", "full": "true"} - assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected - - -def test_issues_request_params(): - stream = Issues(**INIT_ARGS) - expected = {"cursor": "next-page", "statsPeriod": "", "query": ""} - assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected - - -def test_projects_request_params(): - stream = Projects(hostname="sentry.io") - expected = {"cursor": "next-page"} - assert stream.request_params(stream_state=None, next_page_token={"cursor": "next-page"}) == expected - - -def test_project_detail_request_params(): - stream = ProjectDetail(**INIT_ARGS) - expected = {} - assert stream.request_params(stream_state=None, next_page_token=None) == expected From 52d28fe2fc4727018cba2f4f0c76a659bfda89c7 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 10 Aug 2022 18:43:45 -0700 Subject: [PATCH 14/32] handle missing keys --- .../airbyte_cdk/sources/declarative/extractors/jello.py | 5 ++++- .../unit_tests/sources/declarative/extractors/test_jello.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py index f36613e2a56e7..e718f19393ece 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py @@ -40,4 +40,7 @@ def __post_init__(self, options: Mapping[str, Any]): def extract_records(self, response: requests.Response) -> List[Record]: response_body = self.decoder.decode(response) script = self.transform.eval(self.config) - return jello_lib.pyquery(response_body, script) + try: + return jello_lib.pyquery(response_body, script) + except KeyError: + return [] diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py index b9a1ec25322d8..abd464851d5f7 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py @@ -22,6 +22,7 @@ ("test_field_in_config", "_.{{ config['field'] }}", {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), ("test_field_in_options", "_.{{ options['options_field'] }}", {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), ("test_default", "_{{kwargs['field']}}", [{"id": 1}, {"id": 2}], [{"id": 1}, {"id": 2}]), + ("test_field_does_not_exist", "_.record", {"id": 1}, []), ( "test_remove_fields_from_records", "[{k:v for k,v in d.items() if k != 'value_to_remove'} for d in _.data]", From d9252ae2571c0868775c0242760de205048e6519 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 10 Aug 2022 19:21:14 -0700 Subject: [PATCH 15/32] delete stream classes --- .../source-sentry/source_sentry/streams.py | 158 ------------------ 1 file changed, 158 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-sentry/source_sentry/streams.py diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py deleted file mode 100644 index 4e0cb131cae50..0000000000000 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py +++ /dev/null @@ -1,158 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional - -import requests -from airbyte_cdk.sources.streams.http import HttpStream - - -class SentryStream(HttpStream, ABC): - API_VERSION = "0" - URL_TEMPLATE = "https://{hostname}/api/{api_version}/" - primary_key = "id" - - def __init__(self, hostname: str, **kwargs): - super().__init__(**kwargs) - self._url_base = self.URL_TEMPLATE.format(hostname=hostname, api_version=self.API_VERSION) - - @property - def url_base(self) -> str: - return self._url_base - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - return {} - - -class SentryStreamPagination(SentryStream): - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - Expect the link header field to always contain the values ​​for `rel`, `results`, and `cursor`. - If there is actually the next page, rel="next"; results="true"; cursor="". - """ - if response.links["next"]["results"] == "true": - return {"cursor": response.links["next"]["cursor"]} - else: - return None - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - if next_page_token: - params.update(next_page_token) - - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json() - - -class Events(SentryStreamPagination): - """ - Docs: https://docs.sentry.io/api/events/list-a-projects-events/ - """ - - def __init__(self, organization: str, project: str, **kwargs): - super().__init__(**kwargs) - self._organization = organization - self._project = project - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return f"projects/{self._organization}/{self._project}/events/" - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - params.update({"full": "true"}) - - return params - - -class Issues(SentryStreamPagination): - """ - Docs: https://docs.sentry.io/api/events/list-a-projects-issues/ - """ - - def __init__(self, organization: str, project: str, **kwargs): - super().__init__(**kwargs) - self._organization = organization - self._project = project - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return f"projects/{self._organization}/{self._project}/issues/" - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - params.update({"statsPeriod": "", "query": ""}) - - return params - - -class Projects(SentryStreamPagination): - """ - Docs: https://docs.sentry.io/api/projects/list-your-projects/ - """ - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return "projects/" - - -class ProjectDetail(SentryStream): - """ - Docs: https://docs.sentry.io/api/projects/retrieve-a-project/ - """ - - def __init__(self, organization: str, project: str, **kwargs): - super().__init__(**kwargs) - self._organization = organization - self._project = project - - def path( - self, - stream_state: Optional[Mapping[str, Any]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> str: - return f"projects/{self._organization}/{self._project}/" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield response.json() From b582e872cc5415581868498c33b94c9b8ec0f4d6 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 06:58:54 -0700 Subject: [PATCH 16/32] record extractor interface --- .../sources/declarative/extractors/jello.py | 3 +- .../extractors/record_extractor.py | 30 +++++++++++++++++++ .../declarative/extractors/record_selector.py | 6 ++-- .../default_implementation_registry.py | 3 ++ 4 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py index e718f19393ece..89b878daf055a 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py @@ -8,6 +8,7 @@ import requests from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.types import Config, Record from dataclasses_jsonschema import JsonSchemaMixin @@ -15,7 +16,7 @@ @dataclass -class JelloExtractor(JsonSchemaMixin): +class JelloExtractor(RecordExtractor, JsonSchemaMixin): """ Record extractor that evaluates a Jello query to extract records from a decoded response. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py new file mode 100644 index 0000000000000..47b6a00e2014e --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import List + +import requests +from airbyte_cdk.sources.declarative.types import Record + + +@dataclass +class RecordExtractor(ABC): + """ + Responsible for translating an HTTP response into a list of records by extracting records from the response and optionally filtering + records based on a heuristic. + """ + + @abstractmethod + def extract_records( + self, + response: requests.Response, + ) -> List[Record]: + """ + Selects records from the response + :param response: The response to extract the records from + :return: List of Records extracted from the response + """ + pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py index f4a844daa9778..9bb4e6ee34554 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py @@ -7,7 +7,7 @@ import requests from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector -from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState from dataclasses_jsonschema import JsonSchemaMixin @@ -20,11 +20,11 @@ class RecordSelector(HttpSelector, JsonSchemaMixin): records based on a heuristic. Attributes: - extractor (JelloExtractor): The record extractor responsible for extracting records from a response + extractor (RecordExtractor): The record extractor responsible for extracting records from a response record_filter (RecordFilter): The record filter responsible for filtering extracted records """ - extractor: JelloExtractor + extractor: RecordExtractor options: InitVar[Mapping[str, Any]] record_filter: RecordFilter = None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py index f09c00d954e85..1a9862fa59bb1 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py @@ -11,6 +11,8 @@ from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector +from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -58,4 +60,5 @@ SchemaLoader: JsonSchema, Stream: DeclarativeStream, StreamSlicer: SingleSlice, + RecordExtractor: JelloExtractor, } From 1b34c19cf339f60838ebdf8f2a4941a62c2e801b Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 07:16:15 -0700 Subject: [PATCH 17/32] dpath extractor --- .../declarative/extractors/dpath_extractor.py | 43 +++++++++++++++++++ .../extractors/test_dpath_extractor.py | 40 +++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py create mode 100644 airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py new file mode 100644 index 0000000000000..4e392a9f5c3dd --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from dataclasses import InitVar, dataclass +from typing import Any, List, Mapping, Union + +import dpath.util +import requests +from airbyte_cdk.sources.declarative.decoders.decoder import Decoder +from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.types import Config, Record +from dataclasses_jsonschema import JsonSchemaMixin + + +@dataclass +class DpathExtractor(RecordExtractor, JsonSchemaMixin): + """ + Record extractor that evaluates a Jello query to extract records from a decoded response. + + More information on Jello can be found at https://github.com/kellyjonbrazil/jello + + Attributes: + transform (Union[InterpolatedString, str]): The Jello query to evaluate on the decoded response + config (Config): The user-provided configuration as specified by the source's spec + decoder (Decoder): The decoder responsible to transfom the response in a Mapping + """ + + field_pointer: List[Union[InterpolatedString, str]] + config: Config + options: InitVar[Mapping[str, Any]] + decoder: Decoder = JsonDecoder(options={}) + + def __post_init__(self, options: Mapping[str, Any]): + for pointer_index in range(len(self.field_pointer)): + if isinstance(self.field_pointer[pointer_index], str): + self.field_pointer[pointer_index] = InterpolatedString.create(self.field_pointer[pointer_index], options=options) + + def extract_records(self, response: requests.Response) -> List[Record]: + response_body = self.decoder.decode(response) + return dpath.util.get(response_body, [pointer.eval(self.config) for pointer in self.field_pointer], default=[]) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py new file mode 100644 index 0000000000000..cf0371611844c --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py @@ -0,0 +1,40 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json + +import pytest +import requests +from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder +from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor + +config = {"field": "record_array"} +options = {"options_field": "record_array"} + +decoder = JsonDecoder(options={}) + + +@pytest.mark.parametrize( + "test_name, field_pointer, body, expected_records", + [ + ("test_extract_from_array", ["data"], {"data": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), + ("test_nested_field", ["data", "records"], {"data": {"records": [{"id": 1}, {"id": 2}]}}, [{"id": 1}, {"id": 2}]), + ("test_field_in_config", ["{{ config['field'] }}"], {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), + ("test_field_in_options", ["{{ options['options_field'] }}"], {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), + ("test_field_does_not_exist", ["record"], {"id": 1}, []), + ], +) +def test_dpath_extractor(test_name, field_pointer, body, expected_records): + extractor = DpathExtractor(field_pointer=field_pointer, config=config, decoder=decoder, options=options) + + response = create_response(body) + actual_records = extractor.extract_records(response) + + assert actual_records == expected_records + + +def create_response(body): + response = requests.Response() + response._content = json.dumps(body).encode("utf-8") + return response From 3dadc3211b049fa93a4746fb8fd68cac680658f3 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 07:23:20 -0700 Subject: [PATCH 18/32] docstring --- .../declarative/extractors/dpath_extractor.py | 23 +++++++++++++++---- .../parsers/class_types_registry.py | 2 ++ .../default_implementation_registry.py | 4 ++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py index 4e392a9f5c3dd..3859a08708503 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py @@ -18,12 +18,27 @@ @dataclass class DpathExtractor(RecordExtractor, JsonSchemaMixin): """ - Record extractor that evaluates a Jello query to extract records from a decoded response. - - More information on Jello can be found at https://github.com/kellyjonbrazil/jello + Record extractor that searches a decoded response over a path defined as an array of fields. + + Examples of instantiating this transform: + ``` + extractor: + type: DpathExtractor + transform: + - "root" + - "data" + ``` + + ``` + extractor: + type: DpathExtractor + transform: + - "root" + - "{{ options['field'] }}" + ``` Attributes: - transform (Union[InterpolatedString, str]): The Jello query to evaluate on the decoded response + transform (Union[InterpolatedString, str]): Pointer to the field that should be extracted config (Config): The user-provided configuration as specified by the source's spec decoder (Decoder): The decoder responsible to transfom the response in a Mapping """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index ad0c268e1ac15..e075936d289a1 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -7,6 +7,7 @@ from airbyte_cdk.sources.declarative.auth.token import ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream +from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean @@ -46,6 +47,7 @@ "DatetimeStreamSlicer": DatetimeStreamSlicer, "DeclarativeStream": DeclarativeStream, "DefaultErrorHandler": DefaultErrorHandler, + "DpathExtractor": DpathExtractor, "ExponentialBackoffStrategy": ExponentialBackoffStrategy, "HttpRequester": HttpRequester, "InterpolatedBoolean": InterpolatedBoolean, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py index 1a9862fa59bb1..993c49d5f61fb 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py @@ -52,13 +52,13 @@ InterpolatedString: InterpolatedString, MinMaxDatetime: MinMaxDatetime, Paginator: NoPagination, + ParentStreamConfig: ParentStreamConfig, + RecordExtractor: JelloExtractor, RequestOption: RequestOption, RequestOptionsProvider: InterpolatedRequestOptionsProvider, Requester: HttpRequester, Retriever: SimpleRetriever, - ParentStreamConfig: ParentStreamConfig, SchemaLoader: JsonSchema, Stream: DeclarativeStream, StreamSlicer: SingleSlice, - RecordExtractor: JelloExtractor, } From 5e9d9fa80ab31139f29042e0105a9cc47d669bea Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 07:44:52 -0700 Subject: [PATCH 19/32] handle extract root array --- .../declarative/extractors/dpath_extractor.py | 15 ++++++++++++--- .../extractors/test_dpath_extractor.py | 1 + 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py index 3859a08708503..e16a60da2421e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py @@ -24,7 +24,7 @@ class DpathExtractor(RecordExtractor, JsonSchemaMixin): ``` extractor: type: DpathExtractor - transform: + field_pointer: - "root" - "data" ``` @@ -32,11 +32,17 @@ class DpathExtractor(RecordExtractor, JsonSchemaMixin): ``` extractor: type: DpathExtractor - transform: + field_pointer: - "root" - "{{ options['field'] }}" ``` + ``` + extractor: + type: DpathExtractor + field_pointer: [] + ``` + Attributes: transform (Union[InterpolatedString, str]): Pointer to the field that should be extracted config (Config): The user-provided configuration as specified by the source's spec @@ -55,4 +61,7 @@ def __post_init__(self, options: Mapping[str, Any]): def extract_records(self, response: requests.Response) -> List[Record]: response_body = self.decoder.decode(response) - return dpath.util.get(response_body, [pointer.eval(self.config) for pointer in self.field_pointer], default=[]) + if len(self.field_pointer) == 0: + return response_body + else: + return dpath.util.get(response_body, [pointer.eval(self.config) for pointer in self.field_pointer], default=[]) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py index cf0371611844c..0e6ead3c84a23 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py @@ -19,6 +19,7 @@ "test_name, field_pointer, body, expected_records", [ ("test_extract_from_array", ["data"], {"data": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), + ("test_extract_from_root_array", [], [{"id": 1}, {"id": 2}], [{"id": 1}, {"id": 2}]), ("test_nested_field", ["data", "records"], {"data": {"records": [{"id": 1}, {"id": 2}]}}, [{"id": 1}, {"id": 2}]), ("test_field_in_config", ["{{ config['field'] }}"], {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), ("test_field_in_options", ["{{ options['options_field'] }}"], {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), From 8ed13fc65fa3b092f7bcd471353a26b858d1dd58 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 07:48:02 -0700 Subject: [PATCH 20/32] Use dpath extractor --- .../connectors/source-sentry/source_sentry/sentry.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml index 5f3f81a78b1db..567e3aad8ced1 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml @@ -6,8 +6,8 @@ definitions: selector: type: RecordSelector extractor: - type: JelloExtractor - transform: "_ or []" + type: DpathExtractor + field_pointer: [] requester: type: HttpRequester name: "{{ options['name'] }}" From a53760dbfd7dbb9027a44f148dbe1c3f68599506 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 07:48:08 -0700 Subject: [PATCH 21/32] Revert "Merge branch 'alex/selectNoRecords' into alex/configbased-sentry" This reverts commit bad4dd79196c81fb0802ae5f165bdcbd14538b73, reversing changes made to d9252ae2571c0868775c0242760de205048e6519. --- .env | 4 - airbyte-cdk/python/CHANGELOG.md | 5 +- .../declarative/datetime/datetime_parser.py | 38 -- .../declarative/datetime/min_max_datetime.py | 12 +- .../declarative/extractors/dpath_extractor.py | 67 --- .../sources/declarative/extractors/jello.py | 8 +- .../extractors/record_extractor.py | 30 -- .../declarative/extractors/record_selector.py | 8 +- .../parsers/class_types_registry.py | 2 - .../default_implementation_registry.py | 5 +- .../stream_slicers/datetime_stream_slicer.py | 28 +- airbyte-cdk/python/setup.py | 2 +- .../datetime/test_datetime_parser.py | 46 -- .../extractors/test_dpath_extractor.py | 41 -- .../declarative/extractors/test_jello.py | 1 - .../extractors/test_record_selector.py | 14 - .../test_datetime_stream_slicer.py | 9 +- .../main/java/io/airbyte/config/Configs.java | 28 +- .../java/io/airbyte/config/EnvConfigs.java | 25 - .../resources/seed/source_definitions.yaml | 9 +- .../src/main/resources/seed/source_specs.yaml | 21 +- .../ContainerOrchestratorApp.java | 1 + .../source-configuration-based/setup.py.hbs | 2 +- .../connectors/source-file-secure/Dockerfile | 4 +- .../source_file_secure/source.py | 4 +- .../source-hubplanner/.dockerignore | 6 - .../connectors/source-hubplanner/Dockerfile | 16 - .../connectors/source-hubplanner/README.md | 132 ----- .../acceptance-test-config.yml | 26 - .../acceptance-test-docker.sh | 16 - .../connectors/source-hubplanner/build.gradle | 9 - .../integration_tests/__init__.py | 3 - .../integration_tests/acceptance.py | 16 - .../integration_tests/configured_catalog.json | 67 --- .../integration_tests/invalid_config.json | 3 - .../connectors/source-hubplanner/main.py | 13 - .../source-hubplanner/requirements.txt | 2 - .../sample_files/configured_catalog.json | 137 ----- .../connectors/source-hubplanner/setup.py | 29 -- .../source_hubplanner/__init__.py | 8 - .../schemas/billing_rates.json | 45 -- .../source_hubplanner/schemas/bookings.json | 264 ---------- .../source_hubplanner/schemas/clients.json | 48 -- .../source_hubplanner/schemas/events.json | 45 -- .../source_hubplanner/schemas/holidays.json | 43 -- .../source_hubplanner/schemas/projects.json | 490 ------------------ .../source_hubplanner/schemas/resources.json | 212 -------- .../source_hubplanner/source.py | 248 --------- .../source_hubplanner/spec.json | 17 - .../source-hubplanner/unit_tests/__init__.py | 3 - .../unit_tests/test_source.py | 15 - .../unit_tests/test_streams.py | 79 --- .../source-postgres-strict-encrypt/Dockerfile | 2 +- .../connectors/source-postgres/Dockerfile | 2 +- .../relationaldb/StateDecoratingIterator.java | 43 +- .../StateDecoratingIteratorTest.java | 112 +--- .../source-sendgrid/unit_tests/unit_test.py | 24 +- .../CatalogDiffModal.test.tsx | 265 ---------- .../components/DiffAccordion.tsx | 2 +- .../components/DiffFieldTable.tsx | 2 +- .../components/DiffIconBlock.tsx | 12 +- .../components/DiffSection.tsx | 2 +- .../CatalogDiffModal/components/FieldRow.tsx | 10 +- .../components/FieldSection.tsx | 10 +- .../CatalogDiffModal/components/StreamRow.tsx | 2 +- .../CatalogDiffModal/index.stories.tsx | 74 --- .../java/io/airbyte/workers/WorkerApp.java | 32 +- .../workers/process/KubeProcessFactory.java | 2 +- .../sync/NormalizationActivityImpl.java | 1 + .../temporal/sync/SyncWorkflowImpl.java | 25 +- .../workers/temporal/SyncWorkflowTest.java | 3 +- docker-compose.yaml | 4 - docs/integrations/sources/hubplanner.md | 42 -- docs/integrations/sources/postgres.md | 6 +- kube/overlays/dev/.env | 6 - kube/overlays/stable/.env | 5 - kube/resources/worker.yaml | 25 - 77 files changed, 112 insertions(+), 3007 deletions(-) delete mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py delete mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py delete mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py delete mode 100644 airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py delete mode 100644 airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py delete mode 100644 airbyte-integrations/connectors/source-hubplanner/.dockerignore delete mode 100644 airbyte-integrations/connectors/source-hubplanner/Dockerfile delete mode 100644 airbyte-integrations/connectors/source-hubplanner/README.md delete mode 100644 airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml delete mode 100644 airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh delete mode 100644 airbyte-integrations/connectors/source-hubplanner/build.gradle delete mode 100644 airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py delete mode 100644 airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py delete mode 100644 airbyte-integrations/connectors/source-hubplanner/integration_tests/configured_catalog.json delete mode 100644 airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json delete mode 100644 airbyte-integrations/connectors/source-hubplanner/main.py delete mode 100644 airbyte-integrations/connectors/source-hubplanner/requirements.txt delete mode 100644 airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json delete mode 100644 airbyte-integrations/connectors/source-hubplanner/setup.py delete mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py delete mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/billing_rates.json delete mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json delete mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/clients.json delete mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/events.json delete mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/holidays.json delete mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json delete mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json delete mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py delete mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json delete mode 100644 airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py delete mode 100644 airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py delete mode 100644 airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.test.tsx delete mode 100644 airbyte-webapp/src/views/Connection/CatalogDiffModal/index.stories.tsx delete mode 100644 docs/integrations/sources/hubplanner.md diff --git a/.env b/.env index 8efe0cda076ae..a8f8f95d64828 100644 --- a/.env +++ b/.env @@ -67,10 +67,6 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= -NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= -NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= -NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= -NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= ### LOGGING/MONITORING/TRACKING ### TRACKING_STRATEGY=segment diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 00cc78fb6d86e..7f0fa5574a952 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,10 +1,7 @@ # Changelog -## 0.1.73 -- Bugfix: Fix bug in DatetimeStreamSlicer's parsing method - ## 0.1.72 -- Bugfix: Fix bug in DatetimeStreamSlicer's format method +- Bugfix: Fix bug in DatetimeStreamSlicer's parsing method ## 0.1.71 - Refactor declarative package to dataclasses diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py deleted file mode 100644 index f3ed27da3a46f..0000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -import datetime -from typing import Union - - -class DatetimeParser: - """ - Parses and formats datetime objects according to a specified format. - - This class mainly acts as a wrapper to properly handling timestamp formatting through the "%s" directive. - - %s is part of the list of format codes required by the 1989 C standard, but it is unreliable because it always return a datetime in the system's timezone. - Instead of using the directive directly, we can use datetime.fromtimestamp and dt.timestamp() - """ - - def parse(self, date: Union[str, int], format: str, timezone): - # "%s" is a valid (but unreliable) directive for formatting, but not for parsing - # It is defined as - # The number of seconds since the Epoch, 1970-01-01 00:00:00+0000 (UTC). https://man7.org/linux/man-pages/man3/strptime.3.html - # - # The recommended way to parse a date from its timestamp representation is to use datetime.fromtimestamp - # See https://stackoverflow.com/a/4974930 - if format == "%s": - return datetime.datetime.fromtimestamp(int(date), tz=timezone) - else: - return datetime.datetime.strptime(str(date), format).replace(tzinfo=timezone) - - def format(self, dt: datetime.datetime, format: str) -> str: - # strftime("%s") is unreliable because it ignores the time zone information and assumes the time zone of the system it's running on - # It's safer to use the timestamp() method than the %s directive - # See https://stackoverflow.com/a/4974930 - if format == "%s": - return str(int(dt.timestamp())) - else: - return dt.strftime(format) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py index c7b3b498b28ab..0c4b5232cf696 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py @@ -6,7 +6,6 @@ from dataclasses import InitVar, dataclass, field from typing import Any, Mapping, Union -from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from dataclasses_jsonschema import JsonSchemaMixin @@ -41,7 +40,6 @@ class MinMaxDatetime(JsonSchemaMixin): def __post_init__(self, options: Mapping[str, Any]): self.datetime = InterpolatedString.create(self.datetime, options=options or {}) self.timezone = dt.timezone.utc - self._parser = DatetimeParser() self.min_datetime = InterpolatedString.create(self.min_datetime, options=options) if self.min_datetime else None self.max_datetime = InterpolatedString.create(self.max_datetime, options=options) if self.max_datetime else None @@ -59,13 +57,17 @@ def get_datetime(self, config, **additional_options) -> dt.datetime: if not datetime_format: datetime_format = "%Y-%m-%dT%H:%M:%S.%f%z" - time = self._parser.parse(str(self.datetime.eval(config, **additional_options)), datetime_format, self.timezone) + time = dt.datetime.strptime(str(self.datetime.eval(config, **additional_options)), datetime_format).replace(tzinfo=self._timezone) if self.min_datetime: - min_time = self._parser.parse(str(self.min_datetime.eval(config, **additional_options)), datetime_format, self.timezone) + min_time = dt.datetime.strptime(str(self.min_datetime.eval(config, **additional_options)), datetime_format).replace( + tzinfo=self._timezone + ) time = max(time, min_time) if self.max_datetime: - max_time = self._parser.parse(str(self.max_datetime.eval(config, **additional_options)), datetime_format, self.timezone) + max_time = dt.datetime.strptime(str(self.max_datetime.eval(config, **additional_options)), datetime_format).replace( + tzinfo=self._timezone + ) time = min(time, max_time) return time diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py deleted file mode 100644 index e16a60da2421e..0000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/dpath_extractor.py +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -from dataclasses import InitVar, dataclass -from typing import Any, List, Mapping, Union - -import dpath.util -import requests -from airbyte_cdk.sources.declarative.decoders.decoder import Decoder -from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder -from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor -from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.declarative.types import Config, Record -from dataclasses_jsonschema import JsonSchemaMixin - - -@dataclass -class DpathExtractor(RecordExtractor, JsonSchemaMixin): - """ - Record extractor that searches a decoded response over a path defined as an array of fields. - - Examples of instantiating this transform: - ``` - extractor: - type: DpathExtractor - field_pointer: - - "root" - - "data" - ``` - - ``` - extractor: - type: DpathExtractor - field_pointer: - - "root" - - "{{ options['field'] }}" - ``` - - ``` - extractor: - type: DpathExtractor - field_pointer: [] - ``` - - Attributes: - transform (Union[InterpolatedString, str]): Pointer to the field that should be extracted - config (Config): The user-provided configuration as specified by the source's spec - decoder (Decoder): The decoder responsible to transfom the response in a Mapping - """ - - field_pointer: List[Union[InterpolatedString, str]] - config: Config - options: InitVar[Mapping[str, Any]] - decoder: Decoder = JsonDecoder(options={}) - - def __post_init__(self, options: Mapping[str, Any]): - for pointer_index in range(len(self.field_pointer)): - if isinstance(self.field_pointer[pointer_index], str): - self.field_pointer[pointer_index] = InterpolatedString.create(self.field_pointer[pointer_index], options=options) - - def extract_records(self, response: requests.Response) -> List[Record]: - response_body = self.decoder.decode(response) - if len(self.field_pointer) == 0: - return response_body - else: - return dpath.util.get(response_body, [pointer.eval(self.config) for pointer in self.field_pointer], default=[]) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py index 89b878daf055a..f36613e2a56e7 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/jello.py @@ -8,7 +8,6 @@ import requests from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder -from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.types import Config, Record from dataclasses_jsonschema import JsonSchemaMixin @@ -16,7 +15,7 @@ @dataclass -class JelloExtractor(RecordExtractor, JsonSchemaMixin): +class JelloExtractor(JsonSchemaMixin): """ Record extractor that evaluates a Jello query to extract records from a decoded response. @@ -41,7 +40,4 @@ def __post_init__(self, options: Mapping[str, Any]): def extract_records(self, response: requests.Response) -> List[Record]: response_body = self.decoder.decode(response) script = self.transform.eval(self.config) - try: - return jello_lib.pyquery(response_body, script) - except KeyError: - return [] + return jello_lib.pyquery(response_body, script) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py deleted file mode 100644 index 47b6a00e2014e..0000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import List - -import requests -from airbyte_cdk.sources.declarative.types import Record - - -@dataclass -class RecordExtractor(ABC): - """ - Responsible for translating an HTTP response into a list of records by extracting records from the response and optionally filtering - records based on a heuristic. - """ - - @abstractmethod - def extract_records( - self, - response: requests.Response, - ) -> List[Record]: - """ - Selects records from the response - :param response: The response to extract the records from - :return: List of Records extracted from the response - """ - pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py index 9bb4e6ee34554..dd738a69015d9 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py @@ -7,7 +7,7 @@ import requests from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector -from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor +from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState from dataclasses_jsonschema import JsonSchemaMixin @@ -20,11 +20,11 @@ class RecordSelector(HttpSelector, JsonSchemaMixin): records based on a heuristic. Attributes: - extractor (RecordExtractor): The record extractor responsible for extracting records from a response + extractor (JelloExtractor): The record extractor responsible for extracting records from a response record_filter (RecordFilter): The record filter responsible for filtering extracted records """ - extractor: RecordExtractor + extractor: JelloExtractor options: InitVar[Mapping[str, Any]] record_filter: RecordFilter = None @@ -39,8 +39,6 @@ def select_records( next_page_token: Optional[Mapping[str, Any]] = None, ) -> List[Record]: all_records = self.extractor.extract_records(response) - if not all_records: - return [] # Some APIs don't wrap single records in a list if not isinstance(all_records, list): all_records = [all_records] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index e075936d289a1..ad0c268e1ac15 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -7,7 +7,6 @@ from airbyte_cdk.sources.declarative.auth.token import ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream -from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean @@ -47,7 +46,6 @@ "DatetimeStreamSlicer": DatetimeStreamSlicer, "DeclarativeStream": DeclarativeStream, "DefaultErrorHandler": DefaultErrorHandler, - "DpathExtractor": DpathExtractor, "ExponentialBackoffStrategy": ExponentialBackoffStrategy, "HttpRequester": HttpRequester, "InterpolatedBoolean": InterpolatedBoolean, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py index 993c49d5f61fb..f09c00d954e85 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/default_implementation_registry.py @@ -11,8 +11,6 @@ from airbyte_cdk.sources.declarative.decoders.decoder import Decoder from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector -from airbyte_cdk.sources.declarative.extractors.jello import JelloExtractor -from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -52,12 +50,11 @@ InterpolatedString: InterpolatedString, MinMaxDatetime: MinMaxDatetime, Paginator: NoPagination, - ParentStreamConfig: ParentStreamConfig, - RecordExtractor: JelloExtractor, RequestOption: RequestOption, RequestOptionsProvider: InterpolatedRequestOptionsProvider, Requester: HttpRequester, Retriever: SimpleRetriever, + ParentStreamConfig: ParentStreamConfig, SchemaLoader: JsonSchema, Stream: DeclarativeStream, StreamSlicer: SingleSlice, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py index ff08da789638e..c81d11e851298 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py @@ -5,10 +5,9 @@ import datetime import re from dataclasses import InitVar, dataclass, field -from typing import Any, Iterable, Mapping, Optional +from typing import Any, Iterable, Mapping, Optional, Union from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation @@ -78,7 +77,6 @@ def __post_init__(self, options: Mapping[str, Any]): self.cursor_field = InterpolatedString.create(self.cursor_field, options=options) self.stream_slice_field_start = InterpolatedString.create(self.stream_state_field_start or "start_time", options=options) self.stream_slice_field_end = InterpolatedString.create(self.stream_state_field_end or "end_time", options=options) - self._parser = DatetimeParser() # If datetime format is not specified then start/end datetime should inherit it from the stream slicer if not self.start_datetime.datetime_format: @@ -144,12 +142,7 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> start_datetime = max(cursor_datetime, start_datetime) - state_cursor_value = stream_state.get(self.cursor_field.eval(self.config, stream_state=stream_state)) - - if state_cursor_value: - state_date = self.parse_date(state_cursor_value) - else: - state_date = None + state_date = self.parse_date(stream_state.get(self.cursor_field.eval(self.config, stream_state=stream_state))) if state_date: # If the input_state's date is greater than start_datetime, the start of the time window is the state's next day next_date = state_date + datetime.timedelta(days=1) @@ -158,7 +151,13 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> return dates def _format_datetime(self, dt: datetime.datetime): - return self._parser.format(dt, self.datetime_format) + # strftime("%s") is unreliable because it ignores the time zone information and assumes the time zone of the system it's running on + # It's safer to use the timestamp() method than the %s directive + # See https://stackoverflow.com/a/4974930 + if self.datetime_format == "%s": + return str(int(dt.timestamp())) + else: + return dt.strftime(self.datetime_format) def _partition_daterange(self, start, end, step: datetime.timedelta): start_field = self.stream_slice_field_start.eval(self.config) @@ -171,11 +170,14 @@ def _partition_daterange(self, start, end, step: datetime.timedelta): return dates def _get_date(self, cursor_value, default_date: datetime.datetime, comparator) -> datetime.datetime: - cursor_date = cursor_value or default_date + cursor_date = self.parse_date(cursor_value or default_date) return comparator(cursor_date, default_date) - def parse_date(self, date: str) -> datetime.datetime: - return self._parser.parse(date, self.datetime_format, self._timezone) + def parse_date(self, date: Union[str, datetime.datetime]) -> datetime.datetime: + if isinstance(date, str): + return datetime.datetime.strptime(str(date), self.datetime_format).replace(tzinfo=self._timezone) + else: + return date @classmethod def _parse_timedelta(cls, time_str): diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 2c839f7eadcb3..ff3a17f010518 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.1.73", + version="0.1.72", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py deleted file mode 100644 index e4d701fc3e7ab..0000000000000 --- a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -import datetime - -import pytest -from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser - - -@pytest.mark.parametrize( - "test_name, input_date, date_format, expected_output_date", - [ - ( - "test_parse_date_iso", - "2021-01-01T00:00:00.000000+0000", - "%Y-%m-%dT%H:%M:%S.%f%z", - datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), - ), - ( - "test_parse_timestamp", - "1609459200", - "%s", - datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), - ), - ("test_parse_date_number", "20210101", "%Y%m%d", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), - ], -) -def test_parse_date(test_name, input_date, date_format, expected_output_date): - parser = DatetimeParser() - output_date = parser.parse(input_date, date_format, datetime.timezone.utc) - assert expected_output_date == output_date - - -@pytest.mark.parametrize( - "test_name, input_dt, datetimeformat, expected_output", - [ - ("test_format_timestamp", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%s", "1609459200"), - ("test_format_string", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y-%m-%d", "2021-01-01"), - ("test_format_to_number", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y%m%d", "20210101"), - ], -) -def test_format_datetime(test_name, input_dt, datetimeformat, expected_output): - parser = DatetimeParser() - output_date = parser.format(input_dt, datetimeformat) - assert expected_output == output_date diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py deleted file mode 100644 index 0e6ead3c84a23..0000000000000 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -import json - -import pytest -import requests -from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder -from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor - -config = {"field": "record_array"} -options = {"options_field": "record_array"} - -decoder = JsonDecoder(options={}) - - -@pytest.mark.parametrize( - "test_name, field_pointer, body, expected_records", - [ - ("test_extract_from_array", ["data"], {"data": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), - ("test_extract_from_root_array", [], [{"id": 1}, {"id": 2}], [{"id": 1}, {"id": 2}]), - ("test_nested_field", ["data", "records"], {"data": {"records": [{"id": 1}, {"id": 2}]}}, [{"id": 1}, {"id": 2}]), - ("test_field_in_config", ["{{ config['field'] }}"], {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), - ("test_field_in_options", ["{{ options['options_field'] }}"], {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), - ("test_field_does_not_exist", ["record"], {"id": 1}, []), - ], -) -def test_dpath_extractor(test_name, field_pointer, body, expected_records): - extractor = DpathExtractor(field_pointer=field_pointer, config=config, decoder=decoder, options=options) - - response = create_response(body) - actual_records = extractor.extract_records(response) - - assert actual_records == expected_records - - -def create_response(body): - response = requests.Response() - response._content = json.dumps(body).encode("utf-8") - return response diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py index abd464851d5f7..b9a1ec25322d8 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_jello.py @@ -22,7 +22,6 @@ ("test_field_in_config", "_.{{ config['field'] }}", {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), ("test_field_in_options", "_.{{ options['options_field'] }}", {"record_array": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]), ("test_default", "_{{kwargs['field']}}", [{"id": 1}, {"id": 2}], [{"id": 1}, {"id": 2}]), - ("test_field_does_not_exist", "_.record", {"id": 1}, []), ( "test_remove_fields_from_records", "[{k:v for k,v in d.items() if k != 'value_to_remove'} for d in _.data]", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py index f0b94c1cd6fd0..fa2bbfdcd7ce5 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py @@ -43,20 +43,6 @@ {"data": {"id": 1, "created_at": "06-06-21"}}, [{"id": 1, "created_at": "06-06-21"}], ), - ( - "test_no_record", - "_.data", - None, - {"data": []}, - [], - ), - ( - "test_no_record_from_root", - "_", - None, - [], - [], - ), ], ) def test_record_filter(test_name, transform_template, filter_template, body, expected_records): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py index ea83a06ad4499..e2321ad607f21 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py @@ -454,13 +454,13 @@ def test_request_option(test_name, inject_into, field_name, expected_req_params, "%Y-%m-%dT%H:%M:%S.%f%z", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), ), + ("test_parse_date_number", "20210101", "%Y%m%d", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), ( - "test_parse_timestamp", - "1609459200", - "%s", + "test_parse_date_datetime", + datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), + "%Y%m%d", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), ), - ("test_parse_date_number", "20210101", "%Y%m%d", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), ], ) def test_parse_date(test_name, input_date, date_format, expected_output_date): @@ -483,7 +483,6 @@ def test_parse_date(test_name, input_date, date_format, expected_output_date): [ ("test_format_timestamp", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%s", "1609459200"), ("test_format_string", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y-%m-%d", "2021-01-01"), - ("test_format_to_number", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y%m%d", "20210101"), ], ) def test_format_datetime(test_name, input_dt, datetimeformat, expected_output): diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java index 692c396afdfeb..f006052411799 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java @@ -332,41 +332,17 @@ public interface Configs { String getCheckJobMainContainerCpuLimit(); /** - * Define the check job container's minimum RAM usage. Defaults to + * Define the job container's minimum RAM usage. Defaults to * {@link #getJobMainContainerMemoryRequest()} if not set. Internal-use only. */ String getCheckJobMainContainerMemoryRequest(); /** - * Define the check job container's maximum RAM usage. Defaults to + * Define the job container's maximum RAM usage. Defaults to * {@link #getJobMainContainerMemoryLimit()} if not set. Internal-use only. */ String getCheckJobMainContainerMemoryLimit(); - /** - * Define the normalization job container's minimum CPU request. Defaults to - * {@link #getJobMainContainerCpuRequest()} if not set. Internal-use only. - */ - String getNormalizationJobMainContainerCpuRequest(); - - /** - * Define the normalization job container's maximum CPU usage. Defaults to - * {@link #getJobMainContainerCpuLimit()} if not set. Internal-use only. - */ - String getNormalizationJobMainContainerCpuLimit(); - - /** - * Define the normalization job container's minimum RAM usage. Defaults to - * {@link #getJobMainContainerMemoryRequest()} if not set. Internal-use only. - */ - String getNormalizationJobMainContainerMemoryRequest(); - - /** - * Define the normalization job container's maximum RAM usage. Defaults to - * {@link #getJobMainContainerMemoryLimit()} if not set. Internal-use only. - */ - String getNormalizationJobMainContainerMemoryLimit(); - /** * Define one or more Job pod tolerations. Tolerations are separated by ';'. Each toleration * contains k=v pairs mentioning some/all of key, effect, operator and value and separated by `,`. diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java index 9af20b09d7bd3..76e6990230cde 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -153,11 +153,6 @@ public class EnvConfigs implements Configs { static final String CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST = "CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST"; static final String CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT = "CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT"; - static final String NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST = "NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST"; - static final String NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT = "NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT"; - static final String NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST = "NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST"; - static final String NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT = "NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT"; - // defaults private static final String DEFAULT_SPEC_CACHE_BUCKET = "io-airbyte-cloud-spec-cache"; public static final String DEFAULT_JOB_KUBE_NAMESPACE = "default"; @@ -771,26 +766,6 @@ public String getCheckJobMainContainerMemoryLimit() { return getEnvOrDefault(CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT, getJobMainContainerMemoryLimit()); } - @Override - public String getNormalizationJobMainContainerCpuRequest() { - return getEnvOrDefault(NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST, getJobMainContainerCpuRequest()); - } - - @Override - public String getNormalizationJobMainContainerCpuLimit() { - return getEnvOrDefault(NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT, getJobMainContainerCpuLimit()); - } - - @Override - public String getNormalizationJobMainContainerMemoryRequest() { - return getEnvOrDefault(NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST, getJobMainContainerMemoryRequest()); - } - - @Override - public String getNormalizationJobMainContainerMemoryLimit() { - return getEnvOrDefault(NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT, getJobMainContainerMemoryLimit()); - } - @Override public LogConfigs getLogConfigs() { return logConfigs; diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 785db71a42414..444de55646cd8 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -411,13 +411,6 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/hellobaton sourceType: api releaseStage: alpha -- name: Hubplanner - sourceDefinitionId: 8097ceb9-383f-42f6-9f92-d3fd4bcc7689 - dockerRepository: airbyte/source-hubplanner - dockerImageTag: 0.1.0 - documentationUrl: https://docs.airbyte.io/integrations/sources/hubplanner - sourceType: api - releaseStage: alpha - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot @@ -769,7 +762,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.1 + dockerImageTag: 1.0.0 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 079dad4ead84c..3ba04f127e8e4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3703,25 +3703,6 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-hubplanner:0.1.0" - spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/hubplanner" - connectionSpecification: - $schema: "http://json-schema.org/draft-07/schema#" - title: "Hubplanner Spec" - type: "object" - required: - - "api_key" - additionalProperties: true - properties: - api_key: - type: "string" - description: "Hubplanner API key. See https://github.com/hubplanner/API#authentication\ - \ for more details." - airbyte_secret: true - supportsNormalization: false - supportsDBT: false - supported_destination_sync_modes: [] - dockerImage: "airbyte/source-hubspot:0.1.81" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" @@ -7159,7 +7140,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.1" +- dockerImage: "airbyte/source-postgres:1.0.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java index cd0570bfc535d..fb726c0402bd7 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java @@ -196,6 +196,7 @@ private static JobOrchestrator getJobOrchestrator(final Configs configs, final ProcessFactory processFactory, final String application, final FeatureFlags featureFlags) { + return switch (application) { case ReplicationLauncherWorker.REPLICATION -> new ReplicationJobOrchestrator(configs, workerConfigs, processFactory, featureFlags); case NormalizationLauncherWorker.NORMALIZATION -> new NormalizationJobOrchestrator(configs, workerConfigs, processFactory); diff --git a/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs b/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs index e60fbe235ae6b..49b54192a86ec 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1.73", + "airbyte-cdk~=0.1.72", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile index dcb20379a8b85..256d21bdf9b89 100644 --- a/airbyte-integrations/connectors/source-file-secure/Dockerfile +++ b/airbyte-integrations/connectors/source-file-secure/Dockerfile @@ -1,4 +1,4 @@ -FROM airbyte/source-file:0.2.16 +FROM airbyte/source-file:0.2.15 WORKDIR /airbyte/integration_code COPY source_file_secure ./source_file_secure @@ -9,5 +9,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.16 +LABEL io.airbyte.version=0.2.15 LABEL io.airbyte.name=airbyte/source-file-secure diff --git a/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py b/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py index 52fcff45f431a..db36e11d20533 100644 --- a/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py +++ b/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py @@ -36,11 +36,11 @@ class URLFileSecure(ParentURLFile): This connector shouldn't work with local files. """ - def __init__(self, url: str, provider: dict, binary=None, encoding=None): + def __init__(self, url: str, provider: dict): storage_name = provider["storage"].lower() if url.startswith("file://") or storage_name == LOCAL_STORAGE_NAME: raise RuntimeError("the local file storage is not supported by this connector.") - super().__init__(url, provider, binary, encoding) + super().__init__(url, provider) class SourceFileSecure(ParentSourceFile): diff --git a/airbyte-integrations/connectors/source-hubplanner/.dockerignore b/airbyte-integrations/connectors/source-hubplanner/.dockerignore deleted file mode 100644 index beb33d510fd42..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_hubplanner -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-hubplanner/Dockerfile b/airbyte-integrations/connectors/source-hubplanner/Dockerfile deleted file mode 100644 index ba50ba0758fba..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_hubplanner ./source_hubplanner -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-hubplanner diff --git a/airbyte-integrations/connectors/source-hubplanner/README.md b/airbyte-integrations/connectors/source-hubplanner/README.md deleted file mode 100644 index f38554dc57851..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/README.md +++ /dev/null @@ -1,132 +0,0 @@ -# Hubplanner Source - -This is the repository for the Hubplanner source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/hubplanner). - -## Local development - -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-hubplanner:build -``` - -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/hubplanner) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubplanner/spec.json` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. - -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source hubplanner test creds` -and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - -### Locally running the connector docker image - -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-hubplanner:dev -``` - -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-hubplanner:airbyteDocker -``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-hubplanner:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubplanner:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubplanner:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-hubplanner:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json -``` -## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-hubplanner:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-hubplanner:integrationTest -``` - -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list - -### Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml deleted file mode 100644 index 7dece07aad48f..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml +++ /dev/null @@ -1,26 +0,0 @@ -# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-hubplanner:dev -tests: - spec: - - spec_path: "source_hubplanner/spec.json" - connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - discovery: - - config_path: "secrets/config.json" - basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["holidays"] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh deleted file mode 100644 index c51577d10690c..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connectors/source-hubplanner/build.gradle b/airbyte-integrations/connectors/source-hubplanner/build.gradle deleted file mode 100644 index c75aea3d3c219..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-source-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_hubplanner' -} diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py b/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py deleted file mode 100644 index 46b7376756ec6..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py deleted file mode 100644 index 1302b2f57e10e..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -import pytest - -pytest_plugins = ("source_acceptance_test.plugin",) - - -@pytest.fixture(scope="session", autouse=True) -def connector_setup(): - """This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments - yield - # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-hubplanner/integration_tests/configured_catalog.json deleted file mode 100644 index 1404db601f4b8..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/integration_tests/configured_catalog.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "billing_rates", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "bookings", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "clients", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "events", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "holidays", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "projects", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "resources", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json deleted file mode 100644 index 92a71d900e591..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "api_key": "invalid-api-key" -} diff --git a/airbyte-integrations/connectors/source-hubplanner/main.py b/airbyte-integrations/connectors/source-hubplanner/main.py deleted file mode 100644 index 14edc91deb050..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/main.py +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -import sys - -from airbyte_cdk.entrypoint import launch -from source_hubplanner import SourceHubplanner - -if __name__ == "__main__": - source = SourceHubplanner() - launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-hubplanner/requirements.txt b/airbyte-integrations/connectors/source-hubplanner/requirements.txt deleted file mode 100644 index 0411042aa0911..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ --e ../../bases/source-acceptance-test --e . diff --git a/airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json deleted file mode 100644 index 6e00688736fa0..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json +++ /dev/null @@ -1,137 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "billing_rates", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "bookings", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "clients", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "events", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "holidays", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "projects", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "resources", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-hubplanner/setup.py b/airbyte-integrations/connectors/source-hubplanner/setup.py deleted file mode 100644 index cd9e6accd8555..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", -] - -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "pytest-mock~=3.6.1", - "source-acceptance-test", -] - -setup( - name="source_hubplanner", - description="Source implementation for Hubplanner.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py deleted file mode 100644 index 3bcd0c2b2b7ae..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -from .source import SourceHubplanner - -__all__ = ["SourceHubplanner"] diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/billing_rates.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/billing_rates.json deleted file mode 100644 index 810ab001a5bdb..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/billing_rates.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "_id": { - "type": [ - "null", - "string" - ] - }, - "rate": { - "type": [ - "null", - "number" - ] - }, - "metadata": { - "type": [ - "null", - "string" - ] - }, - "createdDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updatedDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "label": { - "type": "string" - }, - "currency": { - "type": "string" - } - } -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json deleted file mode 100644 index e40f7ed91bcc9..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json +++ /dev/null @@ -1,264 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "_id": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "type": { - "type": [ - "null", - "string" - ] - }, - "state": { - "type": [ - "null", - "string" - ] - }, - "allDay": { - "type": [ - "null", - "boolean" - ] - }, - "scale": { - "type": [ - "null", - "string" - ] - }, - "start": { - "type": "string" - }, - "end": { - "type": "string" - }, - "categoryTemplateId": { - "type": [ - "null", - "string" - ] - }, - "categoryName": { - "type": [ - "null", - "string" - ] - }, - "stateValue": { - "type": [ - "null", - "integer" - ] - }, - "resource": { - "type": "string" - }, - "project": { - "type": "string" - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "details": { - "properties": { - "offDaysCount": { - "type": [ - "null", - "integer" - ] - }, - "workDaysCount": { - "type": [ - "null", - "integer" - ] - }, - "holidayCount": { - "type": [ - "null", - "integer" - ] - }, - "workWeekDetails": { - "items": { - "type": [ - "null", - "integer" - ] - }, - "type": [ - "null", - "array" - ] - }, - "bookedMinutes": { - "type": [ - "null", - "integer" - ] - }, - "budgetBookedAmount": { - "type": [ - "null", - "integer" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "createdDate": { - "format": "date-time", - "type": [ - "null", - "string" - ] - }, - "updatedDate": { - "format": "date-time", - "type": [ - "null", - "string" - ] - }, - "metadata": { - "type": [ - "null", - "string" - ] - }, - "customFields": { - "items": { - "properties": { - "_id": { - "type": "string" - }, - "templateId": { - "type": "string" - }, - "templateType": { - "type": "string" - }, - "templateLabel": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "string" - ] - }, - "choices": { - "items": { - "properties": { - "_id": { - "type": "string" - }, - "choiceId": { - "type": "string" - }, - "value": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "bookingRate": { - "properties": { - "external": { - "properties": { - "defaultRateId": { - "type": [ - "null", - "integer" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "internal": { - "properties": { - "defaultRateId": { - "type": [ - "null", - "integer" - ] - } - }, - "type": [ - "null", - "object" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "bookingCreatorId": { - "type": [ - "null", - "string" - ] - }, - "lastUpdatedById": { - "type": [ - "null", - "string" - ] - }, - "backgroundColor": { - "type": [ - "null", - "string" - ] - } - } -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/clients.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/clients.json deleted file mode 100644 index 1b3e5ea8ab566..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/clients.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "_id": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": "string" - }, - "company": { - "type": [ - "null", - "string" - ] - }, - "__v": { - "type": [ - "null", - "integer" - ] - }, - "createdDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updatedDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "deleted": { - "type": [ - "null", - "boolean" - ] - } - } -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/events.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/events.json deleted file mode 100644 index c06d633ef56c5..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/events.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "_id": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": "string" - }, - "createdDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updatedDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "backgroundColor": { - "type": [ - "null", - "string" - ] - }, - "label": { - "type": "string" - }, - "metadata": { - "type": [ - "null", - "string" - ] - } - } -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/holidays.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/holidays.json deleted file mode 100644 index b210ed04dae6e..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/holidays.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "_id": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": "string" - }, - "date": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "metadata": { - "type": [ - "null", - "string" - ] - }, - "createdDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updatedDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - } - } -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json deleted file mode 100644 index f7f7c61eb8ce7..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json +++ /dev/null @@ -1,490 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "_id": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": "string" - }, - "links": { - "properties": {}, - "type": [ - "null", - "object" - ] - }, - "notes": { - "type": [ - "null", - "string" - ] - }, - "createdDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updatedDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "timeEntryEnabled": { - "type": [ - "null", - "boolean" - ] - }, - "timeEntryLocked": { - "type": [ - "null", - "boolean" - ] - }, - "timeEntryApproval": { - "type": [ - "null", - "boolean" - ] - }, - "resourceRates": { - "items": { - "properties": { - "resource": { - "type": [ - "null", - "string" - ] - }, - "companyBillingRateId": { - "type": [ - "null", - "string" - ] - }, - "companyBillingRate": { - "type": [ - "null", - "number" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "includeBookedTimeReports": { - "type": [ - "null", - "boolean" - ] - }, - "includeBookedTimeGrid": { - "type": [ - "null", - "boolean" - ] - }, - "projectManagers": { - "items": { - "type": [ - "null", - "string" - ] - }, - "type": [ - "null", - "array" - ] - }, - "resources": { - "items": { - "type": [ - "null", - "string" - ] - }, - "type": [ - "null", - "array" - ] - }, - "workDays": { - "items": { - "type": [ - "null", - "boolean" - ] - }, - "type": [ - "null", - "array" - ] - }, - "useProjectDays": { - "type": [ - "null", - "boolean" - ] - }, - "budget": { - "properties": { - "hasBudget": { - "type": [ - "null", - "boolean" - ] - }, - "projectHours": { - "properties": { - "active": { - "type": [ - "null", - "boolean" - ] - }, - "hours": { - "type": [ - "null", - "number" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "cashAmount": { - "properties": { - "active": { - "type": [ - "null", - "boolean" - ] - }, - "amount": { - "type": [ - "null", - "number" - ] - }, - "currency": { - "type": [ - "null", - "string" - ] - }, - "billingRate": { - "properties": { - "useDefault": { - "type": [ - "null", - "boolean" - ] - }, - "rate": { - "type": [ - "null", - "number" - ] - }, - "id": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - } - }, - "type": [ - "null", - "object" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "companyBillingRateId": { - "type": [ - "null", - "string" - ] - }, - "budgetHours": { - "type": [ - "null", - "number" - ] - }, - "budgetCashAmount": { - "type": [ - "null", - "number" - ] - }, - "budgetCurrency": { - "type": [ - "null", - "string" - ] - }, - "useStatusColor": { - "type": [ - "null", - "boolean" - ] - }, - "status": { - "type": [ - "null", - "string" - ] - }, - "useProjectDuration": { - "type": [ - "null", - "boolean" - ] - }, - "start": { - "type": [ - "null", - "string" - ] - }, - "end": { - "type": [ - "null", - "string" - ] - }, - "backgroundColor": { - "type": [ - "null", - "string" - ] - }, - "projectCode": { - "type": [ - "null", - "string" - ] - }, - "metadata": { - "type": [ - "null", - "string" - ] - }, - "customFields": { - "items": { - "properties": { - "_id": { - "type": "string" - }, - "templateId": { - "type": "string" - }, - "templateType": { - "type": "string" - }, - "templateLabel": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "string" - ] - }, - "choices": { - "items": { - "properties": { - "_id": { - "type": "string" - }, - "choiceId": { - "type": "string" - }, - "value": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "timeEntryNoteRequired": { - "type": [ - "null", - "boolean" - ] - }, - "projectRate": { - "properties": { - "external": { - "properties": { - "defaultRateId": { - "type": [ - "null", - "string" - ] - }, - "customRates": { - "items": { - "properties": { - "_id": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "string" - ] - }, - "resourceId": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "internal": { - "properties": { - "customRates": { - "items": { - "properties": { - "_id": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "string" - ] - }, - "resourceId": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "defaultRateId": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "tags": { - "items": { - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - } - } -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json deleted file mode 100644 index 996f3435627a8..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json +++ /dev/null @@ -1,212 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "_id": { - "type": [ - "null", - "string" - ] - }, - "email": { - "type": [ - "null", - "string" - ] - }, - "metadata": { - "type": [ - "null", - "string" - ] - }, - "createdDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updatedDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "note": { - "type": [ - "null", - "string" - ] - }, - "firstName": { - "type": "string" - }, - "lastName": { - "type": [ - "null", - "string" - ] - }, - "status": { - "type": [ - "null", - "string" - ] - }, - "role": { - "type": [ - "null", - "string" - ] - }, - "links": { - "properties": {}, - "type": [ - "null", - "object" - ] - }, - "billing": { - "properties": { - "useDefault": { - "type": [ - "null", - "boolean" - ] - }, - "rate": { - "type": [ - "null", - "number" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "useCustomAvailability": { - "type": [ - "null", - "boolean" - ] - }, - "customFields": { - "items": { - "properties": { - "_id": { - "type": "string" - }, - "templateId": { - "type": "string" - }, - "templateType": { - "type": "string" - }, - "templateLabel": { - "type": [ - "null", - "string" - ] - }, - "value": { - "type": [ - "null", - "string" - ] - }, - "choices": { - "items": { - "properties": { - "_id": { - "type": "string" - }, - "choiceId": { - "type": "string" - }, - "value": { - "type": [ - "null", - "string" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "resourceRates": { - "properties": { - "external": { - "items": { - "properties": {}, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - }, - "internal": { - "items": { - "properties": {}, - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - } - }, - "type": [ - "null", - "object" - ] - }, - "isProjectManager": { - "type": [ - "null", - "boolean" - ] - }, - "tags": { - "items": { - "type": [ - "null", - "object" - ] - }, - "type": [ - "null", - "array" - ] - } - } -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py deleted file mode 100644 index 3ca0496d538bf..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py +++ /dev/null @@ -1,248 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple - -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator - - -# Basic full refresh stream -class HubplannerStream(HttpStream, ABC): - - url_base = "https://api.hubplanner.com/v1" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. - - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - - For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a - 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. - The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. - - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - """ - TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. - Usually contains common params e.g. pagination size etc. - """ - return {} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - TODO: Override this method to define how a response is parsed. - :return an iterable containing each record in the response - """ - yield {} - - -# Basic incremental stream -class IncrementalHubplannerStream(HubplannerStream, ABC): - """ - TODO fill in details of this class to implement functionality related to incremental syncs for your connector. - if you do not need to implement incremental sync for any streams, remove this class. - """ - - # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. - state_checkpoint_interval = None - - @property - def cursor_field(self) -> str: - """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - - :return str: The name of the cursor field. - """ - return [] - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - return {} - - -class HubplannerAuthenticator(HttpAuthenticator): - def __init__(self, token: str, auth_header: str = "Authorization"): - self.auth_header = auth_header - self._token = token - - def get_auth_header(self) -> Mapping[str, Any]: - return {self.auth_header: f"{self._token}"} - - -class BillingRates(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/billingRate" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Bookings(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/booking" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Clients(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/client" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Events(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/event" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Holidays(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/holiday" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Projects(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/project" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Resources(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/resource" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -# Source -class SourceHubplanner(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - :param config: the user-input config object conforming to the connector's spec.json - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - - url_base = "https://api.hubplanner.com/v1" - - try: - url = f"{url_base}/project" - - authenticator = HubplannerAuthenticator(token=config["api_key"]) - - session = requests.get(url, headers=authenticator.get_auth_header()) - session.raise_for_status() - - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - authenticator = HubplannerAuthenticator(token=config["api_key"]) - return [ - BillingRates(authenticator=authenticator), - Bookings(authenticator=authenticator), - Clients(authenticator=authenticator), - Events(authenticator=authenticator), - Holidays(authenticator=authenticator), - Projects(authenticator=authenticator), - Resources(authenticator=authenticator), - ] diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json deleted file mode 100644 index aa061822e3221..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubplanner", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Hubplanner Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": true, - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See https://github.com/hubplanner/API#authentication for more details.", - "airbyte_secret": true - } - } - } -} diff --git a/airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py b/airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec6..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py deleted file mode 100644 index 3637179245002..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_hubplanner.source import SourceHubplanner - - -def test_streams(mocker): - source = SourceHubplanner() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 7 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py deleted file mode 100644 index 808f40802b20c..0000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py +++ /dev/null @@ -1,79 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_hubplanner.source import HubplannerStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(HubplannerStream, "path", "v0/example_endpoint") - mocker.patch.object(HubplannerStream, "primary_key", "test_primary_key") - mocker.patch.object(HubplannerStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = HubplannerStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = HubplannerStream() - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = HubplannerStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = HubplannerStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = HubplannerStream() - # TODO: replace this with your expected http request method - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = HubplannerStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = HubplannerStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 51f5cce85e189..5cde4f85fd45a 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.1 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 9dae6b7cff80a..9dfb9767a391a 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.1 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java index 12370d9468b6d..d2880e26a3cdd 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java @@ -13,7 +13,6 @@ import io.airbyte.protocol.models.AirbyteStateMessage; import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.util.Iterator; -import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,16 +27,10 @@ public class StateDecoratingIterator extends AbstractIterator im private final JsonSchemaPrimitive cursorType; private final int stateEmissionFrequency; - private final String initialCursor; private String maxCursor; + private AirbyteMessage intermediateStateMessage; private boolean hasEmittedFinalState; - - // The intermediateStateMessage is set to the latest state message. - // For every stateEmissionFrequency messages, emitIntermediateState is set to true and - // the latest intermediateStateMessage will be emitted. - private int totalRecordCount = 0; - private boolean emitIntermediateState = false; - private AirbyteMessage intermediateStateMessage = null; + private int recordCount; /** * @param stateEmissionFrequency If larger than 0, intermediate states will be emitted for every @@ -56,7 +49,6 @@ public StateDecoratingIterator(final Iterator messageIterator, this.pair = pair; this.cursorField = cursorField; this.cursorType = cursorType; - this.initialCursor = initialCursor; this.maxCursor = initialCursor; this.stateEmissionFrequency = stateEmissionFrequency; } @@ -68,41 +60,36 @@ private String getCursorCandidate(final AirbyteMessage message) { @Override protected AirbyteMessage computeNext() { - if (messageIterator.hasNext()) { - if (emitIntermediateState && intermediateStateMessage != null) { - final AirbyteMessage message = intermediateStateMessage; - intermediateStateMessage = null; - emitIntermediateState = false; - return message; - } - - totalRecordCount++; + if (intermediateStateMessage != null) { + final AirbyteMessage message = intermediateStateMessage; + intermediateStateMessage = null; + return message; + } else if (messageIterator.hasNext()) { + recordCount++; final AirbyteMessage message = messageIterator.next(); if (message.getRecord().getData().hasNonNull(cursorField)) { final String cursorCandidate = getCursorCandidate(message); if (IncrementalUtils.compareCursors(maxCursor, cursorCandidate, cursorType) < 0) { - if (stateEmissionFrequency > 0 && !Objects.equals(maxCursor, initialCursor) && messageIterator.hasNext()) { - // Only emit an intermediate state when it is not the first or last record message, - // because the last state message will be taken care of in a different branch. - intermediateStateMessage = createStateMessage(false); - } maxCursor = cursorCandidate; } } - if (stateEmissionFrequency > 0 && totalRecordCount % stateEmissionFrequency == 0) { - emitIntermediateState = true; + if (stateEmissionFrequency > 0 && recordCount % stateEmissionFrequency == 0) { + // Mark the state as final in case this intermediate state happens to be the last one. + // This is not necessary, but avoid sending the final states twice and prevent any edge case. + final boolean isFinalState = !messageIterator.hasNext(); + intermediateStateMessage = emitStateMessage(isFinalState); } return message; } else if (!hasEmittedFinalState) { - return createStateMessage(true); + return emitStateMessage(true); } else { return endOfData(); } } - public AirbyteMessage createStateMessage(final boolean isFinalState) { + public AirbyteMessage emitStateMessage(final boolean isFinalState) { final AirbyteStateMessage stateMessage = stateManager.updateAndEmit(pair, maxCursor); LOGGER.info("State Report: stream name: {}, original cursor field: {}, original cursor value {}, cursor field: {}, new cursor value: {}", pair, diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java index 474d553d3a1d4..8f16a7d5a11ff 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java @@ -45,18 +45,10 @@ class StateDecoratingIteratorTest { private static final AirbyteMessage RECORD_MESSAGE_2 = createRecordMessage(RECORD_VALUE_2); private static final AirbyteMessage STATE_MESSAGE_2 = createStateMessage(RECORD_VALUE_2); - private static final String RECORD_VALUE_3 = "ghi"; + private static final String RECORD_VALUE_3 = "xyz"; private static final AirbyteMessage RECORD_MESSAGE_3 = createRecordMessage(RECORD_VALUE_3); private static final AirbyteMessage STATE_MESSAGE_3 = createStateMessage(RECORD_VALUE_3); - private static final String RECORD_VALUE_4 = "jkl"; - private static final AirbyteMessage RECORD_MESSAGE_4 = createRecordMessage(RECORD_VALUE_4); - private static final AirbyteMessage STATE_MESSAGE_4 = createStateMessage(RECORD_VALUE_4); - - private static final String RECORD_VALUE_5 = "xyz"; - private static final AirbyteMessage RECORD_MESSAGE_5 = createRecordMessage(RECORD_VALUE_5); - private static final AirbyteMessage STATE_MESSAGE_5 = createStateMessage(RECORD_VALUE_5); - private static AirbyteMessage createRecordMessage(final String recordValue) { return new AirbyteMessage() .withType(Type.RECORD) @@ -81,8 +73,6 @@ void setup() { when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_1)).thenReturn(STATE_MESSAGE_1.getState()); when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_2)).thenReturn(STATE_MESSAGE_2.getState()); when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_3)).thenReturn(STATE_MESSAGE_3.getState()); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_4)).thenReturn(STATE_MESSAGE_4.getState()); - when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_5)).thenReturn(STATE_MESSAGE_5.getState()); when(stateManager.getOriginalCursorField(NAME_NAMESPACE_PAIR)).thenReturn(Optional.empty()); when(stateManager.getOriginalCursor(NAME_NAMESPACE_PAIR)).thenReturn(Optional.empty()); @@ -116,13 +106,13 @@ void testWithInitialCursor() { stateManager, NAME_NAMESPACE_PAIR, UUID_FIELD_NAME, - RECORD_VALUE_5, + RECORD_VALUE_3, JsonSchemaPrimitive.STRING, 0); assertEquals(RECORD_MESSAGE_1, iterator.next()); assertEquals(RECORD_MESSAGE_2, iterator.next()); - assertEquals(STATE_MESSAGE_5, iterator.next()); + assertEquals(STATE_MESSAGE_3, iterator.next()); assertFalse(iterator.hasNext()); } @@ -189,8 +179,8 @@ void testUnicodeNull() { @Test @DisplayName("When initial cursor is null, and emit state for every record") - void testStateEmissionFrequency1() { - messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_3, RECORD_MESSAGE_4, RECORD_MESSAGE_5); + void testStateEmission1() { + messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_3); final StateDecoratingIterator iterator1 = new StateDecoratingIterator( messageIterator, stateManager, @@ -201,27 +191,19 @@ void testStateEmissionFrequency1() { 1); assertEquals(RECORD_MESSAGE_1, iterator1.next()); - // should emit state 1, but it is unclear whether there will be more - // records with the same cursor value, so no state is ready for emission - assertEquals(RECORD_MESSAGE_2, iterator1.next()); - // emit state 1 because it is the latest state ready for emission assertEquals(STATE_MESSAGE_1, iterator1.next()); - assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_2, iterator1.next()); assertEquals(STATE_MESSAGE_2, iterator1.next()); - assertEquals(RECORD_MESSAGE_4, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + // final state message should only be emitted once assertEquals(STATE_MESSAGE_3, iterator1.next()); - assertEquals(RECORD_MESSAGE_5, iterator1.next()); - // state 4 is not emitted because there is no more record and only - // the final state should be emitted at this point; also the final - // state should only be emitted once - assertEquals(STATE_MESSAGE_5, iterator1.next()); assertFalse(iterator1.hasNext()); } @Test @DisplayName("When initial cursor is null, and emit state for every 2 records") - void testStateEmissionFrequency2() { - messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_3, RECORD_MESSAGE_4, RECORD_MESSAGE_5); + void testStateEmission2() { + messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_3); final StateDecoratingIterator iterator1 = new StateDecoratingIterator( messageIterator, stateManager, @@ -233,74 +215,16 @@ void testStateEmissionFrequency2() { assertEquals(RECORD_MESSAGE_1, iterator1.next()); assertEquals(RECORD_MESSAGE_2, iterator1.next()); - // emit state 1 because it is the latest state ready for emission - assertEquals(STATE_MESSAGE_1, iterator1.next()); + assertEquals(STATE_MESSAGE_2, iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); - assertEquals(RECORD_MESSAGE_4, iterator1.next()); - // emit state 3 because it is the latest state ready for emission assertEquals(STATE_MESSAGE_3, iterator1.next()); - assertEquals(RECORD_MESSAGE_5, iterator1.next()); - assertEquals(STATE_MESSAGE_5, iterator1.next()); assertFalse(iterator1.hasNext()); } @Test @DisplayName("When initial cursor is not null") - void testStateEmissionWhenInitialCursorIsNotNull() { - messageIterator = MoreIterators.of(RECORD_MESSAGE_2, RECORD_MESSAGE_3, RECORD_MESSAGE_4, RECORD_MESSAGE_5); - final StateDecoratingIterator iterator1 = new StateDecoratingIterator( - messageIterator, - stateManager, - NAME_NAMESPACE_PAIR, - UUID_FIELD_NAME, - RECORD_VALUE_1, - JsonSchemaPrimitive.STRING, - 1); - - assertEquals(RECORD_MESSAGE_2, iterator1.next()); - assertEquals(RECORD_MESSAGE_3, iterator1.next()); - assertEquals(STATE_MESSAGE_2, iterator1.next()); - assertEquals(RECORD_MESSAGE_4, iterator1.next()); - assertEquals(STATE_MESSAGE_3, iterator1.next()); - assertEquals(RECORD_MESSAGE_5, iterator1.next()); - assertEquals(STATE_MESSAGE_5, iterator1.next()); - assertFalse(iterator1.hasNext()); - } - - /** - * Incremental syncs will sort the table with the cursor field, and emit the max cursor for every N - * records. The purpose is to emit the states frequently, so that if any transient failure occurs - * during a long sync, the next run does not need to start from the beginning, but can resume from - * the last successful intermediate state committed on the destination. The next run will start with - * `cursorField > cursor`. However, it is possible that there are multiple records with the same - * cursor value. If the intermediate state is emitted before all these records have been synced to - * the destination, some of these records may be lost. - *

- * Here is an example: - * - * 

-   * | Record ID | Cursor Field | Other Field | Note                          |
-   * | --------- | ------------ | ----------- | ----------------------------- |
-   * | 1         | F1=16        | F2="abc"    |                               |
-   * | 2         | F1=16        | F2="def"    | <- state emission and failure |
-   * | 3         | F1=16        | F2="ghi"    |                               |
-   *
- * - * If the intermediate state is emitted for record 2 and the sync fails immediately such that the - * cursor value `16` is committed, but only record 1 and 2 are actually synced, the next run will - * start with `F1 > 16` and skip record 3. - *

- * So intermediate state emission should only happen when all records with the same cursor value has - * been synced to destination. Reference: https://github.com/airbytehq/airbyte/issues/15427 - */ - @Test - @DisplayName("When there are multiple records with the same cursor value") - void testStateEmissionForRecordsSharingSameCursorValue() { - messageIterator = MoreIterators.of( - RECORD_MESSAGE_2, RECORD_MESSAGE_2, - RECORD_MESSAGE_3, RECORD_MESSAGE_3, RECORD_MESSAGE_3, - RECORD_MESSAGE_4, - RECORD_MESSAGE_5, RECORD_MESSAGE_5); + void testStateEmission3() { + messageIterator = MoreIterators.of(RECORD_MESSAGE_2, RECORD_MESSAGE_3); final StateDecoratingIterator iterator1 = new StateDecoratingIterator( messageIterator, stateManager, @@ -311,19 +235,9 @@ void testStateEmissionForRecordsSharingSameCursorValue() { 1); assertEquals(RECORD_MESSAGE_2, iterator1.next()); - assertEquals(RECORD_MESSAGE_2, iterator1.next()); - assertEquals(RECORD_MESSAGE_3, iterator1.next()); - // state 2 is the latest state ready for emission because - // all records with the same cursor value have been emitted assertEquals(STATE_MESSAGE_2, iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); - assertEquals(RECORD_MESSAGE_3, iterator1.next()); - assertEquals(RECORD_MESSAGE_4, iterator1.next()); assertEquals(STATE_MESSAGE_3, iterator1.next()); - assertEquals(RECORD_MESSAGE_5, iterator1.next()); - assertEquals(STATE_MESSAGE_4, iterator1.next()); - assertEquals(RECORD_MESSAGE_5, iterator1.next()); - assertEquals(STATE_MESSAGE_5, iterator1.next()); assertFalse(iterator1.hasNext()); } diff --git a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py index c5f8e71ff340b..8f50befe1e23f 100644 --- a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py @@ -2,17 +2,13 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -import unittest from unittest.mock import MagicMock -import pendulum import pytest import requests from airbyte_cdk.logger import AirbyteLogger from source_sendgrid.source import SourceSendgrid -from source_sendgrid.streams import Messages, SendgridStream - -FAKE_NOW = pendulum.DateTime(2022, 1, 1, tzinfo=pendulum.timezone("utc")) +from source_sendgrid.streams import SendgridStream @pytest.fixture(name="sendgrid_stream") @@ -23,13 +19,6 @@ def sendgrid_stream_fixture(mocker) -> SendgridStream: return SendgridStream() # type: ignore -@pytest.fixture() -def mock_pendulum_now(monkeypatch): - pendulum_mock = unittest.mock.MagicMock(wraps=pendulum.now) - pendulum_mock.return_value = FAKE_NOW - monkeypatch.setattr(pendulum, "now", pendulum_mock) - - def test_parse_response_gracefully_handles_nulls(mocker, sendgrid_stream: SendgridStream): response = requests.Response() mocker.patch.object(response, "json", return_value=None) @@ -41,14 +30,3 @@ def test_source_wrong_credentials(): source = SourceSendgrid() status, error = source.check_connection(logger=AirbyteLogger(), config={"apikey": "wrong.api.key123"}) assert not status - - -def test_messages_stream_request_params(mock_pendulum_now): - start_time = 1558359837 - stream = Messages(start_time) - state = {"last_event_time": 1558359000} - request_params = stream.request_params(state) - assert ( - request_params - == "query=last_event_time%20BETWEEN%20TIMESTAMP%20%222019-05-20T06%3A30%3A00Z%22%20AND%20TIMESTAMP%20%222021-12-31T16%3A00%3A00Z%22&limit=1000" - ) diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.test.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.test.tsx deleted file mode 100644 index 8f025a281d077..0000000000000 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.test.tsx +++ /dev/null @@ -1,265 +0,0 @@ -import { cleanup, render, screen } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { IntlProvider } from "react-intl"; - -import { - AirbyteCatalog, - CatalogDiff, - DestinationSyncMode, - StreamTransform, - SyncMode, -} from "core/request/AirbyteClient"; - -import messages from "../../../locales/en.json"; -import { CatalogDiffModal } from "./CatalogDiffModal"; - -const mockCatalogDiff: CatalogDiff = { - transforms: [], -}; - -const removedItems: StreamTransform[] = [ - { - transformType: "remove_stream", - streamDescriptor: { namespace: "apple", name: "dragonfruit" }, - }, - { - transformType: "remove_stream", - streamDescriptor: { namespace: "apple", name: "eclair" }, - }, - { - transformType: "remove_stream", - streamDescriptor: { namespace: "apple", name: "fishcake" }, - }, - { - transformType: "remove_stream", - streamDescriptor: { namespace: "apple", name: "gelatin_mold" }, - }, -]; - -const addedItems: StreamTransform[] = [ - { - transformType: "add_stream", - streamDescriptor: { namespace: "apple", name: "banana" }, - }, - { - transformType: "add_stream", - streamDescriptor: { namespace: "apple", name: "carrot" }, - }, -]; - -const updatedItems: StreamTransform[] = [ - { - transformType: "update_stream", - streamDescriptor: { namespace: "apple", name: "harissa_paste" }, - updateStream: [ - { transformType: "add_field", fieldName: ["users", "phone"] }, - { transformType: "add_field", fieldName: ["users", "email"] }, - { transformType: "remove_field", fieldName: ["users", "lastName"] }, - - { - transformType: "update_field_schema", - fieldName: ["users", "address"], - updateFieldSchema: { oldSchema: { type: "number" }, newSchema: { type: "string" } }, - }, - ], - }, -]; - -const mockCatalog: AirbyteCatalog = { - streams: [ - { - stream: { - namespace: "apple", - name: "banana", - }, - config: { - syncMode: SyncMode.full_refresh, - destinationSyncMode: DestinationSyncMode.overwrite, - }, - }, - { - stream: { - namespace: "apple", - name: "carrot", - }, - config: { - syncMode: SyncMode.full_refresh, - destinationSyncMode: DestinationSyncMode.overwrite, - }, - }, - { - stream: { - namespace: "apple", - name: "dragonfruit", - }, - config: { - syncMode: SyncMode.full_refresh, - destinationSyncMode: DestinationSyncMode.overwrite, - }, - }, - { - stream: { - namespace: "apple", - name: "eclair", - }, - config: { - syncMode: SyncMode.full_refresh, - destinationSyncMode: DestinationSyncMode.overwrite, - }, - }, - { - stream: { - namespace: "apple", - name: "fishcake", - }, - config: { - syncMode: SyncMode.incremental, - destinationSyncMode: DestinationSyncMode.append_dedup, - }, - }, - { - stream: { - namespace: "apple", - name: "gelatin_mold", - }, - config: { - syncMode: SyncMode.incremental, - destinationSyncMode: DestinationSyncMode.append_dedup, - }, - }, - { - stream: { - namespace: "apple", - name: "harissa_paste", - }, - config: { - syncMode: SyncMode.full_refresh, - destinationSyncMode: DestinationSyncMode.overwrite, - }, - }, - ], -}; - -describe("catalog diff modal", () => { - afterEach(cleanup); - beforeEach(() => { - mockCatalogDiff.transforms = []; - }); - - test("it renders the correct section for each type of transform", () => { - mockCatalogDiff.transforms.push(...addedItems, ...removedItems, ...updatedItems); - - render( - - { - return null; - }} - /> - - ); - - /** - * tests for: - * - proper sections being created - * - syncmode string is only rendered for removed streams - */ - - const newStreamsTable = screen.getByRole("table", { name: /new streams/ }); - expect(newStreamsTable).toBeInTheDocument(); - - const newStreamRow = screen.getByRole("row", { name: "apple banana" }); - expect(newStreamRow).toBeInTheDocument(); - - const newStreamRowWithSyncMode = screen.queryByRole("row", { name: "apple carrot incremental | append_dedup" }); - expect(newStreamRowWithSyncMode).not.toBeInTheDocument(); - - const removedStreamsTable = screen.getByRole("table", { name: /removed streams/ }); - expect(removedStreamsTable).toBeInTheDocument(); - - const removedStreamRowWithSyncMode = screen.getByRole("row", { - name: "apple dragonfruit full_refresh | overwrite", - }); - expect(removedStreamRowWithSyncMode).toBeInTheDocument(); - - const updatedStreamsSection = screen.getByRole("list", { name: /table with changes/ }); - expect(updatedStreamsSection).toBeInTheDocument(); - - const updatedStreamRowWithSyncMode = screen.queryByRole("row", { - name: "apple harissa_paste full_refresh | overwrite", - }); - expect(updatedStreamRowWithSyncMode).not.toBeInTheDocument(); - }); - - test("added fields are not rendered when not in the diff", () => { - mockCatalogDiff.transforms.push(...removedItems, ...updatedItems); - - render( - - { - return null; - }} - /> - - ); - - const newStreamsTable = screen.queryByRole("table", { name: /new streams/ }); - expect(newStreamsTable).not.toBeInTheDocument(); - }); - - test("removed fields are not rendered when not in the diff", () => { - mockCatalogDiff.transforms.push(...addedItems, ...updatedItems); - - render( - - { - return null; - }} - /> - - ); - - const removedStreamsTable = screen.queryByRole("table", { name: /removed streams/ }); - expect(removedStreamsTable).not.toBeInTheDocument(); - }); - - test("changed streams accordion opens/closes on clicking the description row", () => { - mockCatalogDiff.transforms.push(...addedItems, ...updatedItems); - - render( - - { - return null; - }} - /> - - ); - - const accordionHeader = screen.getByRole("button", { name: /toggle accordion/ }); - - expect(accordionHeader).toBeInTheDocument(); - - const nullAccordionBody = screen.queryByRole("table", { name: /removed fields/ }); - expect(nullAccordionBody).not.toBeInTheDocument(); - - userEvent.click(accordionHeader); - const openAccordionBody = screen.getByRole("table", { name: /removed fields/ }); - expect(openAccordionBody).toBeInTheDocument(); - - userEvent.click(accordionHeader); - const nullAccordionBodyAgain = screen.queryByRole("table", { name: /removed fields/ }); - expect(nullAccordionBodyAgain).not.toBeInTheDocument(); - mockCatalogDiff.transforms = []; - }); -}); diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx index 112563dba001e..d42a051c03b99 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx @@ -23,7 +23,7 @@ export const DiffAccordion: React.FC = ({ streamTransform }) {({ open }) => ( <> - + = ({ fieldTransforms, diffVerb }) => { return ( - +
diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffIconBlock.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffIconBlock.tsx index a206a18fd5f2d..453f23443dcff 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffIconBlock.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffIconBlock.tsx @@ -19,13 +19,13 @@ export const DiffIconBlock: React.FC = ({ newCount, removedC num={removedCount} color="red" light - ariaLabel={`${formatMessage( + ariaLabel={`${removedCount} ${formatMessage( { id: "connection.updateSchema.removed", }, { value: removedCount, - item: formatMessage({ id: "connection.updateSchema.field" }, { count: removedCount }), + item: formatMessage({ id: "field" }, { values: { count: removedCount } }), } )}`} /> @@ -35,13 +35,13 @@ export const DiffIconBlock: React.FC = ({ newCount, removedC num={newCount} color="green" light - ariaLabel={`${formatMessage( + ariaLabel={`${newCount} ${formatMessage( { id: "connection.updateSchema.new", }, { value: newCount, - item: formatMessage({ id: "connection.updateSchema.field" }, { count: newCount }), + item: formatMessage({ id: "field" }, { values: { count: newCount } }), } )}`} /> @@ -51,13 +51,13 @@ export const DiffIconBlock: React.FC = ({ newCount, removedC num={changedCount} color="blue" light - ariaLabel={`${formatMessage( + ariaLabel={`${changedCount} ${formatMessage( { id: "connection.updateSchema.changed", }, { value: changedCount, - item: formatMessage({ id: "connection.updateSchema.field" }, { count: changedCount }), + item: formatMessage({ id: "field" }, { values: { count: changedCount } }), } )}`} /> diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffSection.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffSection.tsx index cd82648fdee96..47981779d2d9b 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffSection.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffSection.tsx @@ -31,7 +31,7 @@ export const DiffSection: React.FC = ({ streams, catalog, diff
- +
+ ); diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.tsx index f6d150fb80b16..4c03fcf8a4e7b 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.tsx @@ -32,15 +32,7 @@ export const FieldSection: React.FC = ({ streams, diffVerb })
{streams.length > 0 && ( -
    +
      {streams.map((stream) => { return (
    • diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/StreamRow.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/StreamRow.tsx index de077b279de8c..2931ed5ed644b 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/StreamRow.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/StreamRow.tsx @@ -45,7 +45,7 @@ export const StreamRow: React.FC = ({ streamTransform, syncMode, )}
- + {" "} ); diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/index.stories.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/index.stories.tsx deleted file mode 100644 index ca2181e4a465b..0000000000000 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/index.stories.tsx +++ /dev/null @@ -1,74 +0,0 @@ -import { ComponentStory, ComponentMeta } from "@storybook/react"; -import { FormattedMessage } from "react-intl"; - -import Modal from "components/Modal"; - -import { CatalogDiffModal } from "./CatalogDiffModal"; - -export default { - title: "Ui/CatalogDiffModal", - component: CatalogDiffModal, -} as ComponentMeta; - -const Template: ComponentStory = (args) => { - return ( - }> - { - return null; - }} - /> - - ); -}; - -export const Primary = Template.bind({}); - -Primary.args = { - catalogDiff: { - transforms: [ - { - transformType: "update_stream", - streamDescriptor: { namespace: "apple", name: "harissa_paste" }, - updateStream: [ - { transformType: "add_field", fieldName: ["users", "phone"] }, - { transformType: "add_field", fieldName: ["users", "email"] }, - { transformType: "remove_field", fieldName: ["users", "lastName"] }, - - { - transformType: "update_field_schema", - fieldName: ["users", "address"], - updateFieldSchema: { oldSchema: { type: "number" }, newSchema: { type: "string" } }, - }, - ], - }, - { - transformType: "add_stream", - streamDescriptor: { namespace: "apple", name: "banana" }, - }, - { - transformType: "add_stream", - streamDescriptor: { namespace: "apple", name: "carrot" }, - }, - { - transformType: "remove_stream", - streamDescriptor: { namespace: "apple", name: "dragonfruit" }, - }, - { - transformType: "remove_stream", - streamDescriptor: { namespace: "apple", name: "eclair" }, - }, - { - transformType: "remove_stream", - streamDescriptor: { namespace: "apple", name: "fishcake" }, - }, - { - transformType: "remove_stream", - streamDescriptor: { namespace: "apple", name: "gelatin_mold" }, - }, - ], - }, - catalog: { streams: [] }, -}; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java index c4c75b3346e47..c82d3785a7ef1 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java @@ -223,10 +223,9 @@ private void registerConnectionManager(final WorkerFactory factory) { private void registerSync(final WorkerFactory factory) { final ReplicationActivityImpl replicationActivity = getReplicationActivityImpl(replicationWorkerConfigs, replicationProcessFactory); - // Note that the configuration injected here is for the normalization orchestrator, and not the - // normalization pod itself. - // Configuration for the normalization pod is injected via the SyncWorkflowImpl. - final NormalizationActivityImpl normalizationActivity = getNormalizationActivityImpl(defaultWorkerConfigs, defaultProcessFactory); + final NormalizationActivityImpl normalizationActivity = getNormalizationActivityImpl( + defaultWorkerConfigs, + defaultProcessFactory); final DbtTransformationActivityImpl dbtTransformationActivity = getDbtActivityImpl( defaultWorkerConfigs, @@ -315,15 +314,6 @@ private DbtTransformationActivityImpl getDbtActivityImpl(final WorkerConfigs wor airbyteVersion); } - /** - * Return either a docker or kubernetes process factory depending on the environment in - * {@link WorkerConfigs} - * - * @param configs used to determine which process factory to create. - * @param workerConfigs used to create the process factory. - * @return either a {@link DockerProcessFactory} or a {@link KubeProcessFactory}. - * @throws IOException - */ private static ProcessFactory getJobProcessFactory(final Configs configs, final WorkerConfigs workerConfigs) throws IOException { if (configs.getWorkerEnvironment() == Configs.WorkerEnvironment.KUBERNETES) { final KubernetesClient fabricClient = new DefaultKubernetesClient(); @@ -351,14 +341,14 @@ private static WorkerOptions getWorkerOptions(final int max) { .build(); } - public record ContainerOrchestratorConfig( - String namespace, - DocumentStoreClient documentStoreClient, - KubernetesClient kubernetesClient, - String secretName, - String secretMountPath, - String containerOrchestratorImage, - String googleApplicationCredentials) {} + public static record ContainerOrchestratorConfig( + String namespace, + DocumentStoreClient documentStoreClient, + KubernetesClient kubernetesClient, + String secretName, + String secretMountPath, + String containerOrchestratorImage, + String googleApplicationCredentials) {} static Optional getContainerOrchestratorConfig(final Configs configs) { if (configs.getContainerOrchestratorEnabled()) { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java index 99227c82cc8fc..72c52a97c1805 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java @@ -97,7 +97,7 @@ public Process create( try { // used to differentiate source and destination processes with the same id and attempt final String podName = ProcessFactory.createProcessName(imageName, jobType, jobId, attempt, KUBE_NAME_LEN_LIMIT); - LOGGER.info("Attempting to start pod = {} for {} with resources {}", podName, imageName, resourceRequirements); + LOGGER.info("Attempting to start pod = {} for {}", podName, imageName); final int stdoutLocalPort = KubePortManagerSingleton.getInstance().take(); LOGGER.info("{} stdoutLocalPort = {}", podName, stdoutLocalPort); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index 36c08b4cd7acc..fb07a62db54f3 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -128,6 +128,7 @@ private CheckedSupplier, Except throws IOException { final var jobScope = jobPersistence.getJob(Long.parseLong(jobRunConfig.getJobId())).getScope(); final var connectionId = UUID.fromString(jobScope); + return () -> new NormalizationLauncherWorker( connectionId, destinationLauncherConfig, diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index 0ca129ae780a2..e60ff71409bf0 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -4,12 +4,9 @@ package io.airbyte.workers.temporal.sync; -import io.airbyte.config.Configs; -import io.airbyte.config.EnvConfigs; import io.airbyte.config.NormalizationInput; import io.airbyte.config.NormalizationSummary; import io.airbyte.config.OperatorDbtInput; -import io.airbyte.config.ResourceRequirements; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOperation; import io.airbyte.config.StandardSyncOperation.OperatorType; @@ -58,8 +55,11 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, if (syncInput.getOperationSequence() != null && !syncInput.getOperationSequence().isEmpty()) { for (final StandardSyncOperation standardSyncOperation : syncInput.getOperationSequence()) { if (standardSyncOperation.getOperatorType() == OperatorType.NORMALIZATION) { - final Configs configs = new EnvConfigs(); - final NormalizationInput normalizationInput = generateNormalizationInput(syncInput, syncOutput, configs); + final NormalizationInput normalizationInput = new NormalizationInput() + .withDestinationConfiguration(syncInput.getDestinationConfiguration()) + .withCatalog(syncOutput.getOutputCatalog()) + .withResourceRequirements(syncInput.getDestinationResourceRequirements()); + final NormalizationSummary normalizationSummary = normalizationActivity.normalize(jobRunConfig, destinationLauncherConfig, normalizationInput); syncOutput = syncOutput.withNormalizationSummary(normalizationSummary); @@ -80,19 +80,4 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, return syncOutput; } - private NormalizationInput generateNormalizationInput(final StandardSyncInput syncInput, - final StandardSyncOutput syncOutput, - final Configs configs) { - final ResourceRequirements resourceReqs = new ResourceRequirements() - .withCpuRequest(configs.getNormalizationJobMainContainerCpuRequest()) - .withCpuLimit(configs.getNormalizationJobMainContainerCpuLimit()) - .withMemoryRequest(configs.getNormalizationJobMainContainerMemoryRequest()) - .withMemoryLimit(configs.getNormalizationJobMainContainerMemoryLimit()); - - return new NormalizationInput() - .withDestinationConfiguration(syncInput.getDestinationConfiguration()) - .withCatalog(syncOutput.getOutputCatalog()) - .withResourceRequirements(resourceReqs); - } - } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/SyncWorkflowTest.java index dc979922d58a2..2258e03ecc99e 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/SyncWorkflowTest.java @@ -100,8 +100,7 @@ public void setUp() { normalizationInput = new NormalizationInput() .withDestinationConfiguration(syncInput.getDestinationConfiguration()) - .withCatalog(syncInput.getCatalog()) - .withResourceRequirements(new ResourceRequirements()); + .withCatalog(syncInput.getCatalog()); operatorDbtInput = new OperatorDbtInput() .withDestinationConfiguration(syncInput.getDestinationConfiguration()) diff --git a/docker-compose.yaml b/docker-compose.yaml index 61502aa9a1c6b..13566de5e479b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -77,10 +77,6 @@ services: - MAX_DISCOVER_WORKERS=${MAX_DISCOVER_WORKERS} - MAX_SPEC_WORKERS=${MAX_SPEC_WORKERS} - MAX_SYNC_WORKERS=${MAX_SYNC_WORKERS} - - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT=${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT} - - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST=${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST} - - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT=${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT} - - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST=${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST} - SECRET_PERSISTENCE=${SECRET_PERSISTENCE} - SYNC_JOB_MAX_ATTEMPTS=${SYNC_JOB_MAX_ATTEMPTS} - SYNC_JOB_MAX_TIMEOUT_DAYS=${SYNC_JOB_MAX_TIMEOUT_DAYS} diff --git a/docs/integrations/sources/hubplanner.md b/docs/integrations/sources/hubplanner.md deleted file mode 100644 index 916e7dbd8fa35..0000000000000 --- a/docs/integrations/sources/hubplanner.md +++ /dev/null @@ -1,42 +0,0 @@ -# Hubplanner - -Hubplanner is a tool to plan, schedule, report and manage your entire team. - -## Prerequisites -* Create the API Key to access your data in Hubplanner. - -## Airbyte OSS -* API Key - -## Airbyte Cloud -* Comming Soon. - - -## Setup guide -### For Airbyte OSS: - -1. Access https://.hubplanner.com/settings#api or access the panel in left side Integrations/Hub Planner API -2. Click in Generate Key - -## Supported sync modes - -The Okta source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): - - Full Refresh - -## Supported Streams - -- [Billing Rates](https://github.com/hubplanner/API/blob/master/Sections/billingrate.md) -- [Bookings](https://github.com/hubplanner/API/blob/master/Sections/bookings.md) -- [Clients](https://github.com/hubplanner/API/blob/master/Sections/clients.md) -- [Events](https://github.com/hubplanner/API/blob/master/Sections/events.md) -- [Holidays](https://github.com/hubplanner/API/blob/master/Sections/holidays.md) -- [Projects](https://github.com/hubplanner/API/blob/master/Sections/project.md) -- [Resources](https://github.com/hubplanner/API/blob/master/Sections/resource.md) - - -## Changelog - -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------| - -| 0.1.0 | 2021-08-10 | [12145](https://github.com/airbytehq/airbyte/pull/12145) | Initial Release | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 58e8a83b0b2c4..1f66efb72ac31 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -371,20 +371,18 @@ Possible solutions include: | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------------------| -| 1.0.1 | 2022-08-10 | [15496](https://github.com/airbytehq/airbyte/pull/15496) | Fix state emission in incremental sync | -| | 2022-08-10 | [15481](https://github.com/airbytehq/airbyte/pull/15481) | Fix data handling from WAL logs in CDC mode | | 1.0.0 | 2022-08-05 | [15380](https://github.com/airbytehq/airbyte/pull/15380) | Change connector label to generally_available | | 0.4.44 | 2022-08-05 | [15342](https://github.com/airbytehq/airbyte/pull/15342) | Adjust titles and descriptions in spec.json | | 0.4.43 | 2022-08-03 | [15226](https://github.com/airbytehq/airbyte/pull/15226) | Make connectionTimeoutMs configurable through JDBC url parameters | | 0.4.42 | 2022-08-03 | [15273](https://github.com/airbytehq/airbyte/pull/15273) | Fix a bug in `0.4.36` and correctly parse the CDC initial record waiting time | | 0.4.41 | 2022-08-03 | [15077](https://github.com/airbytehq/airbyte/pull/15077) | Sync data from beginning if the LSN is no longer valid in CDC | -| | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently (⛔ this version has a bug; use `1.0.1` instead) | +| | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently | | 0.4.40 | 2022-08-03 | [15187](https://github.com/airbytehq/airbyte/pull/15187) | Add support for BCE dates/timestamps | | | 2022-08-03 | [14534](https://github.com/airbytehq/airbyte/pull/14534) | Align regular and CDC integration tests and data mappers | | 0.4.39 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | | 0.4.38 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | | 0.4.37 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead) | +| 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead) | | 0.4.35 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | | 0.4.34 | 2022-07-17 | [13840](https://github.com/airbytehq/airbyte/pull/13840) | Added the ability to connect using different SSL modes and SSL certificates. | | 0.4.33 | 2022-07-14 | [14586](https://github.com/airbytehq/airbyte/pull/14586) | Validate source JDBC url parameters | diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env index 230efad8e4ece..d622254798359 100644 --- a/kube/overlays/dev/.env +++ b/kube/overlays/dev/.env @@ -56,11 +56,6 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= -NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= -NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= -NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= -NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= - # Worker pod tolerations, annotations and node selectors JOB_KUBE_TOLERATIONS= JOB_KUBE_ANNOTATIONS= @@ -71,7 +66,6 @@ JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY= # Launch a separate pod to orchestrate sync steps CONTAINER_ORCHESTRATOR_ENABLED=true -CONTAINER_ORCHESTRATOR_IMAGE= # Open Telemetry Configuration METRIC_CLIENT= diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index e1b01c39f2704..b48947558c899 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -56,11 +56,6 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= -NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= -NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= -NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= -NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= - # Worker pod tolerations, annotations and node selectors JOB_KUBE_TOLERATIONS= JOB_KUBE_ANNOTATIONS= diff --git a/kube/resources/worker.yaml b/kube/resources/worker.yaml index 02ceacb266e26..f5906abd1914a 100644 --- a/kube/resources/worker.yaml +++ b/kube/resources/worker.yaml @@ -119,26 +119,6 @@ spec: configMapKeyRef: name: airbyte-env key: JOB_MAIN_CONTAINER_MEMORY_LIMIT - - name: NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST - valueFrom: - configMapKeyRef: - name: airbyte-env - key: NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST - - name: NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT - valueFrom: - configMapKeyRef: - name: airbyte-env - key: NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT - - name: NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST - valueFrom: - configMapKeyRef: - name: airbyte-env - key: NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST - - name: NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT - valueFrom: - configMapKeyRef: - name: airbyte-env - key: NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT - name: S3_LOG_BUCKET valueFrom: configMapKeyRef: @@ -230,11 +210,6 @@ spec: configMapKeyRef: name: airbyte-env key: CONTAINER_ORCHESTRATOR_ENABLED - - name: CONTAINER_ORCHESTRATOR_IMAGE - valueFrom: - configMapKeyRef: - name: airbyte-env - key: CONTAINER_ORCHESTRATOR_IMAGE - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION valueFrom: configMapKeyRef: From aa0d77773b44904194e38a89bbae2b38cf4e3a55 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 14:26:04 -0700 Subject: [PATCH 22/32] reset to master --- airbyte-cdk/python/CHANGELOG.md | 2 +- .../declarative/datetime/datetime_parser.py | 38 +++ .../declarative/datetime/min_max_datetime.py | 12 +- .../stream_slicers/datetime_stream_slicer.py | 28 +- .../datetime/test_datetime_parser.py | 46 +++ .../test_datetime_stream_slicer.py | 9 +- .../main/java/io/airbyte/config/Configs.java | 28 +- .../java/io/airbyte/config/EnvConfigs.java | 25 ++ .../resources/seed/source_definitions.yaml | 9 +- .../src/main/resources/seed/source_specs.yaml | 21 +- .../ContainerOrchestratorApp.java | 1 - .../source-configuration-based/setup.py.hbs | 2 +- .../CatalogDiffModal.test.tsx | 265 ++++++++++++++++++ .../components/DiffAccordion.tsx | 2 +- .../components/DiffFieldTable.tsx | 2 +- .../components/DiffIconBlock.tsx | 12 +- .../components/DiffSection.tsx | 2 +- .../CatalogDiffModal/components/FieldRow.tsx | 10 +- .../components/FieldSection.tsx | 10 +- .../CatalogDiffModal/components/StreamRow.tsx | 2 +- .../CatalogDiffModal/index.stories.tsx | 74 +++++ .../java/io/airbyte/workers/WorkerApp.java | 32 ++- .../workers/process/KubeProcessFactory.java | 2 +- .../sync/NormalizationActivityImpl.java | 1 - .../temporal/sync/SyncWorkflowImpl.java | 25 +- .../workers/temporal/SyncWorkflowTest.java | 3 +- docker-compose.yaml | 4 + 27 files changed, 599 insertions(+), 68 deletions(-) create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py create mode 100644 airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py create mode 100644 airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.test.tsx create mode 100644 airbyte-webapp/src/views/Connection/CatalogDiffModal/index.stories.tsx diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 589d954b8290d..05f9deed11e50 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -7,7 +7,7 @@ - Bugfix: Fix bug in DatetimeStreamSlicer's parsing method ## 0.1.72 -- Bugfix: Fix bug in DatetimeStreamSlicer's parsing method +- Bugfix: Fix bug in DatetimeStreamSlicer's format method ## 0.1.71 - Refactor declarative package to dataclasses diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py new file mode 100644 index 0000000000000..f3ed27da3a46f --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/datetime_parser.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import datetime +from typing import Union + + +class DatetimeParser: + """ + Parses and formats datetime objects according to a specified format. + + This class mainly acts as a wrapper to properly handling timestamp formatting through the "%s" directive. + + %s is part of the list of format codes required by the 1989 C standard, but it is unreliable because it always return a datetime in the system's timezone. + Instead of using the directive directly, we can use datetime.fromtimestamp and dt.timestamp() + """ + + def parse(self, date: Union[str, int], format: str, timezone): + # "%s" is a valid (but unreliable) directive for formatting, but not for parsing + # It is defined as + # The number of seconds since the Epoch, 1970-01-01 00:00:00+0000 (UTC). https://man7.org/linux/man-pages/man3/strptime.3.html + # + # The recommended way to parse a date from its timestamp representation is to use datetime.fromtimestamp + # See https://stackoverflow.com/a/4974930 + if format == "%s": + return datetime.datetime.fromtimestamp(int(date), tz=timezone) + else: + return datetime.datetime.strptime(str(date), format).replace(tzinfo=timezone) + + def format(self, dt: datetime.datetime, format: str) -> str: + # strftime("%s") is unreliable because it ignores the time zone information and assumes the time zone of the system it's running on + # It's safer to use the timestamp() method than the %s directive + # See https://stackoverflow.com/a/4974930 + if format == "%s": + return str(int(dt.timestamp())) + else: + return dt.strftime(format) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py index 0c4b5232cf696..c7b3b498b28ab 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/datetime/min_max_datetime.py @@ -6,6 +6,7 @@ from dataclasses import InitVar, dataclass, field from typing import Any, Mapping, Union +from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from dataclasses_jsonschema import JsonSchemaMixin @@ -40,6 +41,7 @@ class MinMaxDatetime(JsonSchemaMixin): def __post_init__(self, options: Mapping[str, Any]): self.datetime = InterpolatedString.create(self.datetime, options=options or {}) self.timezone = dt.timezone.utc + self._parser = DatetimeParser() self.min_datetime = InterpolatedString.create(self.min_datetime, options=options) if self.min_datetime else None self.max_datetime = InterpolatedString.create(self.max_datetime, options=options) if self.max_datetime else None @@ -57,17 +59,13 @@ def get_datetime(self, config, **additional_options) -> dt.datetime: if not datetime_format: datetime_format = "%Y-%m-%dT%H:%M:%S.%f%z" - time = dt.datetime.strptime(str(self.datetime.eval(config, **additional_options)), datetime_format).replace(tzinfo=self._timezone) + time = self._parser.parse(str(self.datetime.eval(config, **additional_options)), datetime_format, self.timezone) if self.min_datetime: - min_time = dt.datetime.strptime(str(self.min_datetime.eval(config, **additional_options)), datetime_format).replace( - tzinfo=self._timezone - ) + min_time = self._parser.parse(str(self.min_datetime.eval(config, **additional_options)), datetime_format, self.timezone) time = max(time, min_time) if self.max_datetime: - max_time = dt.datetime.strptime(str(self.max_datetime.eval(config, **additional_options)), datetime_format).replace( - tzinfo=self._timezone - ) + max_time = self._parser.parse(str(self.max_datetime.eval(config, **additional_options)), datetime_format, self.timezone) time = min(time, max_time) return time diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py index c81d11e851298..ff08da789638e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/stream_slicers/datetime_stream_slicer.py @@ -5,9 +5,10 @@ import datetime import re from dataclasses import InitVar, dataclass, field -from typing import Any, Iterable, Mapping, Optional, Union +from typing import Any, Iterable, Mapping, Optional from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation @@ -77,6 +78,7 @@ def __post_init__(self, options: Mapping[str, Any]): self.cursor_field = InterpolatedString.create(self.cursor_field, options=options) self.stream_slice_field_start = InterpolatedString.create(self.stream_state_field_start or "start_time", options=options) self.stream_slice_field_end = InterpolatedString.create(self.stream_state_field_end or "end_time", options=options) + self._parser = DatetimeParser() # If datetime format is not specified then start/end datetime should inherit it from the stream slicer if not self.start_datetime.datetime_format: @@ -142,7 +144,12 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> start_datetime = max(cursor_datetime, start_datetime) - state_date = self.parse_date(stream_state.get(self.cursor_field.eval(self.config, stream_state=stream_state))) + state_cursor_value = stream_state.get(self.cursor_field.eval(self.config, stream_state=stream_state)) + + if state_cursor_value: + state_date = self.parse_date(state_cursor_value) + else: + state_date = None if state_date: # If the input_state's date is greater than start_datetime, the start of the time window is the state's next day next_date = state_date + datetime.timedelta(days=1) @@ -151,13 +158,7 @@ def stream_slices(self, sync_mode: SyncMode, stream_state: Mapping[str, Any]) -> return dates def _format_datetime(self, dt: datetime.datetime): - # strftime("%s") is unreliable because it ignores the time zone information and assumes the time zone of the system it's running on - # It's safer to use the timestamp() method than the %s directive - # See https://stackoverflow.com/a/4974930 - if self.datetime_format == "%s": - return str(int(dt.timestamp())) - else: - return dt.strftime(self.datetime_format) + return self._parser.format(dt, self.datetime_format) def _partition_daterange(self, start, end, step: datetime.timedelta): start_field = self.stream_slice_field_start.eval(self.config) @@ -170,14 +171,11 @@ def _partition_daterange(self, start, end, step: datetime.timedelta): return dates def _get_date(self, cursor_value, default_date: datetime.datetime, comparator) -> datetime.datetime: - cursor_date = self.parse_date(cursor_value or default_date) + cursor_date = cursor_value or default_date return comparator(cursor_date, default_date) - def parse_date(self, date: Union[str, datetime.datetime]) -> datetime.datetime: - if isinstance(date, str): - return datetime.datetime.strptime(str(date), self.datetime_format).replace(tzinfo=self._timezone) - else: - return date + def parse_date(self, date: str) -> datetime.datetime: + return self._parser.parse(date, self.datetime_format, self._timezone) @classmethod def _parse_timedelta(cls, time_str): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py new file mode 100644 index 0000000000000..e4d701fc3e7ab --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import datetime + +import pytest +from airbyte_cdk.sources.declarative.datetime.datetime_parser import DatetimeParser + + +@pytest.mark.parametrize( + "test_name, input_date, date_format, expected_output_date", + [ + ( + "test_parse_date_iso", + "2021-01-01T00:00:00.000000+0000", + "%Y-%m-%dT%H:%M:%S.%f%z", + datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), + ), + ( + "test_parse_timestamp", + "1609459200", + "%s", + datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), + ), + ("test_parse_date_number", "20210101", "%Y%m%d", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), + ], +) +def test_parse_date(test_name, input_date, date_format, expected_output_date): + parser = DatetimeParser() + output_date = parser.parse(input_date, date_format, datetime.timezone.utc) + assert expected_output_date == output_date + + +@pytest.mark.parametrize( + "test_name, input_dt, datetimeformat, expected_output", + [ + ("test_format_timestamp", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%s", "1609459200"), + ("test_format_string", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y-%m-%d", "2021-01-01"), + ("test_format_to_number", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y%m%d", "20210101"), + ], +) +def test_format_datetime(test_name, input_dt, datetimeformat, expected_output): + parser = DatetimeParser() + output_date = parser.format(input_dt, datetimeformat) + assert expected_output == output_date diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py index e2321ad607f21..ea83a06ad4499 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_datetime_stream_slicer.py @@ -454,13 +454,13 @@ def test_request_option(test_name, inject_into, field_name, expected_req_params, "%Y-%m-%dT%H:%M:%S.%f%z", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), ), - ("test_parse_date_number", "20210101", "%Y%m%d", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), ( - "test_parse_date_datetime", - datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), - "%Y%m%d", + "test_parse_timestamp", + "1609459200", + "%s", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), ), + ("test_parse_date_number", "20210101", "%Y%m%d", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), ], ) def test_parse_date(test_name, input_date, date_format, expected_output_date): @@ -483,6 +483,7 @@ def test_parse_date(test_name, input_date, date_format, expected_output_date): [ ("test_format_timestamp", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%s", "1609459200"), ("test_format_string", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y-%m-%d", "2021-01-01"), + ("test_format_to_number", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y%m%d", "20210101"), ], ) def test_format_datetime(test_name, input_dt, datetimeformat, expected_output): diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java index f006052411799..692c396afdfeb 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java @@ -332,17 +332,41 @@ public interface Configs { String getCheckJobMainContainerCpuLimit(); /** - * Define the job container's minimum RAM usage. Defaults to + * Define the check job container's minimum RAM usage. Defaults to * {@link #getJobMainContainerMemoryRequest()} if not set. Internal-use only. */ String getCheckJobMainContainerMemoryRequest(); /** - * Define the job container's maximum RAM usage. Defaults to + * Define the check job container's maximum RAM usage. Defaults to * {@link #getJobMainContainerMemoryLimit()} if not set. Internal-use only. */ String getCheckJobMainContainerMemoryLimit(); + /** + * Define the normalization job container's minimum CPU request. Defaults to + * {@link #getJobMainContainerCpuRequest()} if not set. Internal-use only. + */ + String getNormalizationJobMainContainerCpuRequest(); + + /** + * Define the normalization job container's maximum CPU usage. Defaults to + * {@link #getJobMainContainerCpuLimit()} if not set. Internal-use only. + */ + String getNormalizationJobMainContainerCpuLimit(); + + /** + * Define the normalization job container's minimum RAM usage. Defaults to + * {@link #getJobMainContainerMemoryRequest()} if not set. Internal-use only. + */ + String getNormalizationJobMainContainerMemoryRequest(); + + /** + * Define the normalization job container's maximum RAM usage. Defaults to + * {@link #getJobMainContainerMemoryLimit()} if not set. Internal-use only. + */ + String getNormalizationJobMainContainerMemoryLimit(); + /** * Define one or more Job pod tolerations. Tolerations are separated by ';'. Each toleration * contains k=v pairs mentioning some/all of key, effect, operator and value and separated by `,`. diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java index 76e6990230cde..9af20b09d7bd3 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -153,6 +153,11 @@ public class EnvConfigs implements Configs { static final String CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST = "CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST"; static final String CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT = "CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT"; + static final String NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST = "NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST"; + static final String NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT = "NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT"; + static final String NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST = "NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST"; + static final String NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT = "NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT"; + // defaults private static final String DEFAULT_SPEC_CACHE_BUCKET = "io-airbyte-cloud-spec-cache"; public static final String DEFAULT_JOB_KUBE_NAMESPACE = "default"; @@ -766,6 +771,26 @@ public String getCheckJobMainContainerMemoryLimit() { return getEnvOrDefault(CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT, getJobMainContainerMemoryLimit()); } + @Override + public String getNormalizationJobMainContainerCpuRequest() { + return getEnvOrDefault(NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST, getJobMainContainerCpuRequest()); + } + + @Override + public String getNormalizationJobMainContainerCpuLimit() { + return getEnvOrDefault(NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT, getJobMainContainerCpuLimit()); + } + + @Override + public String getNormalizationJobMainContainerMemoryRequest() { + return getEnvOrDefault(NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST, getJobMainContainerMemoryRequest()); + } + + @Override + public String getNormalizationJobMainContainerMemoryLimit() { + return getEnvOrDefault(NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT, getJobMainContainerMemoryLimit()); + } + @Override public LogConfigs getLogConfigs() { return logConfigs; diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e18c312ed8f02..107f98e1ffd7d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -411,6 +411,13 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/hellobaton sourceType: api releaseStage: alpha +- name: Hubplanner + sourceDefinitionId: 8097ceb9-383f-42f6-9f92-d3fd4bcc7689 + dockerRepository: airbyte/source-hubplanner + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/hubplanner + sourceType: api + releaseStage: alpha - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot @@ -770,7 +777,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 1.0.0 + dockerImageTag: 1.0.1 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 128d88bf324e8..2eb3d36376ad9 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3703,6 +3703,25 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-hubplanner:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/hubplanner" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Hubplanner Spec" + type: "object" + required: + - "api_key" + additionalProperties: true + properties: + api_key: + type: "string" + description: "Hubplanner API key. See https://github.com/hubplanner/API#authentication\ + \ for more details." + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-hubspot:0.1.81" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" @@ -7218,7 +7237,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:1.0.0" +- dockerImage: "airbyte/source-postgres:1.0.1" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java index fb726c0402bd7..cd0570bfc535d 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java @@ -196,7 +196,6 @@ private static JobOrchestrator getJobOrchestrator(final Configs configs, final ProcessFactory processFactory, final String application, final FeatureFlags featureFlags) { - return switch (application) { case ReplicationLauncherWorker.REPLICATION -> new ReplicationJobOrchestrator(configs, workerConfigs, processFactory, featureFlags); case NormalizationLauncherWorker.NORMALIZATION -> new NormalizationJobOrchestrator(configs, workerConfigs, processFactory); diff --git a/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs b/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs index 49b54192a86ec..e60fbe235ae6b 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/setup.py.hbs @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1.72", + "airbyte-cdk~=0.1.73", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.test.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.test.tsx new file mode 100644 index 0000000000000..8f025a281d077 --- /dev/null +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/CatalogDiffModal.test.tsx @@ -0,0 +1,265 @@ +import { cleanup, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { IntlProvider } from "react-intl"; + +import { + AirbyteCatalog, + CatalogDiff, + DestinationSyncMode, + StreamTransform, + SyncMode, +} from "core/request/AirbyteClient"; + +import messages from "../../../locales/en.json"; +import { CatalogDiffModal } from "./CatalogDiffModal"; + +const mockCatalogDiff: CatalogDiff = { + transforms: [], +}; + +const removedItems: StreamTransform[] = [ + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "dragonfruit" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "eclair" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "fishcake" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "gelatin_mold" }, + }, +]; + +const addedItems: StreamTransform[] = [ + { + transformType: "add_stream", + streamDescriptor: { namespace: "apple", name: "banana" }, + }, + { + transformType: "add_stream", + streamDescriptor: { namespace: "apple", name: "carrot" }, + }, +]; + +const updatedItems: StreamTransform[] = [ + { + transformType: "update_stream", + streamDescriptor: { namespace: "apple", name: "harissa_paste" }, + updateStream: [ + { transformType: "add_field", fieldName: ["users", "phone"] }, + { transformType: "add_field", fieldName: ["users", "email"] }, + { transformType: "remove_field", fieldName: ["users", "lastName"] }, + + { + transformType: "update_field_schema", + fieldName: ["users", "address"], + updateFieldSchema: { oldSchema: { type: "number" }, newSchema: { type: "string" } }, + }, + ], + }, +]; + +const mockCatalog: AirbyteCatalog = { + streams: [ + { + stream: { + namespace: "apple", + name: "banana", + }, + config: { + syncMode: SyncMode.full_refresh, + destinationSyncMode: DestinationSyncMode.overwrite, + }, + }, + { + stream: { + namespace: "apple", + name: "carrot", + }, + config: { + syncMode: SyncMode.full_refresh, + destinationSyncMode: DestinationSyncMode.overwrite, + }, + }, + { + stream: { + namespace: "apple", + name: "dragonfruit", + }, + config: { + syncMode: SyncMode.full_refresh, + destinationSyncMode: DestinationSyncMode.overwrite, + }, + }, + { + stream: { + namespace: "apple", + name: "eclair", + }, + config: { + syncMode: SyncMode.full_refresh, + destinationSyncMode: DestinationSyncMode.overwrite, + }, + }, + { + stream: { + namespace: "apple", + name: "fishcake", + }, + config: { + syncMode: SyncMode.incremental, + destinationSyncMode: DestinationSyncMode.append_dedup, + }, + }, + { + stream: { + namespace: "apple", + name: "gelatin_mold", + }, + config: { + syncMode: SyncMode.incremental, + destinationSyncMode: DestinationSyncMode.append_dedup, + }, + }, + { + stream: { + namespace: "apple", + name: "harissa_paste", + }, + config: { + syncMode: SyncMode.full_refresh, + destinationSyncMode: DestinationSyncMode.overwrite, + }, + }, + ], +}; + +describe("catalog diff modal", () => { + afterEach(cleanup); + beforeEach(() => { + mockCatalogDiff.transforms = []; + }); + + test("it renders the correct section for each type of transform", () => { + mockCatalogDiff.transforms.push(...addedItems, ...removedItems, ...updatedItems); + + render( + + { + return null; + }} + /> + + ); + + /** + * tests for: + * - proper sections being created + * - syncmode string is only rendered for removed streams + */ + + const newStreamsTable = screen.getByRole("table", { name: /new streams/ }); + expect(newStreamsTable).toBeInTheDocument(); + + const newStreamRow = screen.getByRole("row", { name: "apple banana" }); + expect(newStreamRow).toBeInTheDocument(); + + const newStreamRowWithSyncMode = screen.queryByRole("row", { name: "apple carrot incremental | append_dedup" }); + expect(newStreamRowWithSyncMode).not.toBeInTheDocument(); + + const removedStreamsTable = screen.getByRole("table", { name: /removed streams/ }); + expect(removedStreamsTable).toBeInTheDocument(); + + const removedStreamRowWithSyncMode = screen.getByRole("row", { + name: "apple dragonfruit full_refresh | overwrite", + }); + expect(removedStreamRowWithSyncMode).toBeInTheDocument(); + + const updatedStreamsSection = screen.getByRole("list", { name: /table with changes/ }); + expect(updatedStreamsSection).toBeInTheDocument(); + + const updatedStreamRowWithSyncMode = screen.queryByRole("row", { + name: "apple harissa_paste full_refresh | overwrite", + }); + expect(updatedStreamRowWithSyncMode).not.toBeInTheDocument(); + }); + + test("added fields are not rendered when not in the diff", () => { + mockCatalogDiff.transforms.push(...removedItems, ...updatedItems); + + render( + + { + return null; + }} + /> + + ); + + const newStreamsTable = screen.queryByRole("table", { name: /new streams/ }); + expect(newStreamsTable).not.toBeInTheDocument(); + }); + + test("removed fields are not rendered when not in the diff", () => { + mockCatalogDiff.transforms.push(...addedItems, ...updatedItems); + + render( + + { + return null; + }} + /> + + ); + + const removedStreamsTable = screen.queryByRole("table", { name: /removed streams/ }); + expect(removedStreamsTable).not.toBeInTheDocument(); + }); + + test("changed streams accordion opens/closes on clicking the description row", () => { + mockCatalogDiff.transforms.push(...addedItems, ...updatedItems); + + render( + + { + return null; + }} + /> + + ); + + const accordionHeader = screen.getByRole("button", { name: /toggle accordion/ }); + + expect(accordionHeader).toBeInTheDocument(); + + const nullAccordionBody = screen.queryByRole("table", { name: /removed fields/ }); + expect(nullAccordionBody).not.toBeInTheDocument(); + + userEvent.click(accordionHeader); + const openAccordionBody = screen.getByRole("table", { name: /removed fields/ }); + expect(openAccordionBody).toBeInTheDocument(); + + userEvent.click(accordionHeader); + const nullAccordionBodyAgain = screen.queryByRole("table", { name: /removed fields/ }); + expect(nullAccordionBodyAgain).not.toBeInTheDocument(); + mockCatalogDiff.transforms = []; + }); +}); diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx index d42a051c03b99..112563dba001e 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffAccordion.tsx @@ -23,7 +23,7 @@ export const DiffAccordion: React.FC = ({ streamTransform }) {({ open }) => ( <> - + = ({ fieldTransforms, diffVerb }) => { return ( -
diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldRow.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldRow.tsx index 3f8d715c59065..1c77e9a7281af 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldRow.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldRow.tsx @@ -28,7 +28,7 @@ export const FieldRow: React.FC = ({ transform }) => { [styles.mod]: diffType === "update", }); - const contentStyle = classnames(styles.content, styles.cell, { + const contentStyle = classnames(styles.content, { [styles.add]: diffType === "add", [styles.remove]: diffType === "remove", [styles.update]: diffType === "update", @@ -50,16 +50,16 @@ export const FieldRow: React.FC = ({ transform }) => { )} -
+
{fieldName} - -
+
{oldType && newType && ( {oldType} {newType} )} - +
{namespace}{itemName}{itemName}{diffVerb === "removed" && syncMode && }
+
diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffIconBlock.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffIconBlock.tsx index 453f23443dcff..a206a18fd5f2d 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffIconBlock.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffIconBlock.tsx @@ -19,13 +19,13 @@ export const DiffIconBlock: React.FC = ({ newCount, removedC num={removedCount} color="red" light - ariaLabel={`${removedCount} ${formatMessage( + ariaLabel={`${formatMessage( { id: "connection.updateSchema.removed", }, { value: removedCount, - item: formatMessage({ id: "field" }, { values: { count: removedCount } }), + item: formatMessage({ id: "connection.updateSchema.field" }, { count: removedCount }), } )}`} /> @@ -35,13 +35,13 @@ export const DiffIconBlock: React.FC = ({ newCount, removedC num={newCount} color="green" light - ariaLabel={`${newCount} ${formatMessage( + ariaLabel={`${formatMessage( { id: "connection.updateSchema.new", }, { value: newCount, - item: formatMessage({ id: "field" }, { values: { count: newCount } }), + item: formatMessage({ id: "connection.updateSchema.field" }, { count: newCount }), } )}`} /> @@ -51,13 +51,13 @@ export const DiffIconBlock: React.FC = ({ newCount, removedC num={changedCount} color="blue" light - ariaLabel={`${changedCount} ${formatMessage( + ariaLabel={`${formatMessage( { id: "connection.updateSchema.changed", }, { value: changedCount, - item: formatMessage({ id: "field" }, { values: { count: changedCount } }), + item: formatMessage({ id: "connection.updateSchema.field" }, { count: changedCount }), } )}`} /> diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffSection.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffSection.tsx index 47981779d2d9b..cd82648fdee96 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffSection.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/DiffSection.tsx @@ -31,7 +31,7 @@ export const DiffSection: React.FC = ({ streams, catalog, diff
- +
- + ); diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.tsx index 4c03fcf8a4e7b..f6d150fb80b16 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldSection.tsx @@ -32,7 +32,15 @@ export const FieldSection: React.FC = ({ streams, diffVerb })
{streams.length > 0 && ( -
    +
      {streams.map((stream) => { return (
    • diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/StreamRow.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/StreamRow.tsx index 2931ed5ed644b..de077b279de8c 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/StreamRow.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/StreamRow.tsx @@ -45,7 +45,7 @@ export const StreamRow: React.FC = ({ streamTransform, syncMode, )}
- {" "} + ); diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/index.stories.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/index.stories.tsx new file mode 100644 index 0000000000000..ca2181e4a465b --- /dev/null +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/index.stories.tsx @@ -0,0 +1,74 @@ +import { ComponentStory, ComponentMeta } from "@storybook/react"; +import { FormattedMessage } from "react-intl"; + +import Modal from "components/Modal"; + +import { CatalogDiffModal } from "./CatalogDiffModal"; + +export default { + title: "Ui/CatalogDiffModal", + component: CatalogDiffModal, +} as ComponentMeta; + +const Template: ComponentStory = (args) => { + return ( + }> + { + return null; + }} + /> + + ); +}; + +export const Primary = Template.bind({}); + +Primary.args = { + catalogDiff: { + transforms: [ + { + transformType: "update_stream", + streamDescriptor: { namespace: "apple", name: "harissa_paste" }, + updateStream: [ + { transformType: "add_field", fieldName: ["users", "phone"] }, + { transformType: "add_field", fieldName: ["users", "email"] }, + { transformType: "remove_field", fieldName: ["users", "lastName"] }, + + { + transformType: "update_field_schema", + fieldName: ["users", "address"], + updateFieldSchema: { oldSchema: { type: "number" }, newSchema: { type: "string" } }, + }, + ], + }, + { + transformType: "add_stream", + streamDescriptor: { namespace: "apple", name: "banana" }, + }, + { + transformType: "add_stream", + streamDescriptor: { namespace: "apple", name: "carrot" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "dragonfruit" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "eclair" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "fishcake" }, + }, + { + transformType: "remove_stream", + streamDescriptor: { namespace: "apple", name: "gelatin_mold" }, + }, + ], + }, + catalog: { streams: [] }, +}; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java index c82d3785a7ef1..c4c75b3346e47 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java @@ -223,9 +223,10 @@ private void registerConnectionManager(final WorkerFactory factory) { private void registerSync(final WorkerFactory factory) { final ReplicationActivityImpl replicationActivity = getReplicationActivityImpl(replicationWorkerConfigs, replicationProcessFactory); - final NormalizationActivityImpl normalizationActivity = getNormalizationActivityImpl( - defaultWorkerConfigs, - defaultProcessFactory); + // Note that the configuration injected here is for the normalization orchestrator, and not the + // normalization pod itself. + // Configuration for the normalization pod is injected via the SyncWorkflowImpl. + final NormalizationActivityImpl normalizationActivity = getNormalizationActivityImpl(defaultWorkerConfigs, defaultProcessFactory); final DbtTransformationActivityImpl dbtTransformationActivity = getDbtActivityImpl( defaultWorkerConfigs, @@ -314,6 +315,15 @@ private DbtTransformationActivityImpl getDbtActivityImpl(final WorkerConfigs wor airbyteVersion); } + /** + * Return either a docker or kubernetes process factory depending on the environment in + * {@link WorkerConfigs} + * + * @param configs used to determine which process factory to create. + * @param workerConfigs used to create the process factory. + * @return either a {@link DockerProcessFactory} or a {@link KubeProcessFactory}. + * @throws IOException + */ private static ProcessFactory getJobProcessFactory(final Configs configs, final WorkerConfigs workerConfigs) throws IOException { if (configs.getWorkerEnvironment() == Configs.WorkerEnvironment.KUBERNETES) { final KubernetesClient fabricClient = new DefaultKubernetesClient(); @@ -341,14 +351,14 @@ private static WorkerOptions getWorkerOptions(final int max) { .build(); } - public static record ContainerOrchestratorConfig( - String namespace, - DocumentStoreClient documentStoreClient, - KubernetesClient kubernetesClient, - String secretName, - String secretMountPath, - String containerOrchestratorImage, - String googleApplicationCredentials) {} + public record ContainerOrchestratorConfig( + String namespace, + DocumentStoreClient documentStoreClient, + KubernetesClient kubernetesClient, + String secretName, + String secretMountPath, + String containerOrchestratorImage, + String googleApplicationCredentials) {} static Optional getContainerOrchestratorConfig(final Configs configs) { if (configs.getContainerOrchestratorEnabled()) { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java index 72c52a97c1805..99227c82cc8fc 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java @@ -97,7 +97,7 @@ public Process create( try { // used to differentiate source and destination processes with the same id and attempt final String podName = ProcessFactory.createProcessName(imageName, jobType, jobId, attempt, KUBE_NAME_LEN_LIMIT); - LOGGER.info("Attempting to start pod = {} for {}", podName, imageName); + LOGGER.info("Attempting to start pod = {} for {} with resources {}", podName, imageName, resourceRequirements); final int stdoutLocalPort = KubePortManagerSingleton.getInstance().take(); LOGGER.info("{} stdoutLocalPort = {}", podName, stdoutLocalPort); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index fb07a62db54f3..36c08b4cd7acc 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -128,7 +128,6 @@ private CheckedSupplier, Except throws IOException { final var jobScope = jobPersistence.getJob(Long.parseLong(jobRunConfig.getJobId())).getScope(); final var connectionId = UUID.fromString(jobScope); - return () -> new NormalizationLauncherWorker( connectionId, destinationLauncherConfig, diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java index e60ff71409bf0..0ca129ae780a2 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/SyncWorkflowImpl.java @@ -4,9 +4,12 @@ package io.airbyte.workers.temporal.sync; +import io.airbyte.config.Configs; +import io.airbyte.config.EnvConfigs; import io.airbyte.config.NormalizationInput; import io.airbyte.config.NormalizationSummary; import io.airbyte.config.OperatorDbtInput; +import io.airbyte.config.ResourceRequirements; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOperation; import io.airbyte.config.StandardSyncOperation.OperatorType; @@ -55,11 +58,8 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, if (syncInput.getOperationSequence() != null && !syncInput.getOperationSequence().isEmpty()) { for (final StandardSyncOperation standardSyncOperation : syncInput.getOperationSequence()) { if (standardSyncOperation.getOperatorType() == OperatorType.NORMALIZATION) { - final NormalizationInput normalizationInput = new NormalizationInput() - .withDestinationConfiguration(syncInput.getDestinationConfiguration()) - .withCatalog(syncOutput.getOutputCatalog()) - .withResourceRequirements(syncInput.getDestinationResourceRequirements()); - + final Configs configs = new EnvConfigs(); + final NormalizationInput normalizationInput = generateNormalizationInput(syncInput, syncOutput, configs); final NormalizationSummary normalizationSummary = normalizationActivity.normalize(jobRunConfig, destinationLauncherConfig, normalizationInput); syncOutput = syncOutput.withNormalizationSummary(normalizationSummary); @@ -80,4 +80,19 @@ public StandardSyncOutput run(final JobRunConfig jobRunConfig, return syncOutput; } + private NormalizationInput generateNormalizationInput(final StandardSyncInput syncInput, + final StandardSyncOutput syncOutput, + final Configs configs) { + final ResourceRequirements resourceReqs = new ResourceRequirements() + .withCpuRequest(configs.getNormalizationJobMainContainerCpuRequest()) + .withCpuLimit(configs.getNormalizationJobMainContainerCpuLimit()) + .withMemoryRequest(configs.getNormalizationJobMainContainerMemoryRequest()) + .withMemoryLimit(configs.getNormalizationJobMainContainerMemoryLimit()); + + return new NormalizationInput() + .withDestinationConfiguration(syncInput.getDestinationConfiguration()) + .withCatalog(syncOutput.getOutputCatalog()) + .withResourceRequirements(resourceReqs); + } + } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/SyncWorkflowTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/SyncWorkflowTest.java index 2258e03ecc99e..dc979922d58a2 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/SyncWorkflowTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/SyncWorkflowTest.java @@ -100,7 +100,8 @@ public void setUp() { normalizationInput = new NormalizationInput() .withDestinationConfiguration(syncInput.getDestinationConfiguration()) - .withCatalog(syncInput.getCatalog()); + .withCatalog(syncInput.getCatalog()) + .withResourceRequirements(new ResourceRequirements()); operatorDbtInput = new OperatorDbtInput() .withDestinationConfiguration(syncInput.getDestinationConfiguration()) diff --git a/docker-compose.yaml b/docker-compose.yaml index 13566de5e479b..61502aa9a1c6b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -77,6 +77,10 @@ services: - MAX_DISCOVER_WORKERS=${MAX_DISCOVER_WORKERS} - MAX_SPEC_WORKERS=${MAX_SPEC_WORKERS} - MAX_SYNC_WORKERS=${MAX_SYNC_WORKERS} + - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT=${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT} + - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST=${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST} + - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT=${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT} + - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST=${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST} - SECRET_PERSISTENCE=${SECRET_PERSISTENCE} - SYNC_JOB_MAX_ATTEMPTS=${SYNC_JOB_MAX_ATTEMPTS} - SYNC_JOB_MAX_TIMEOUT_DAYS=${SYNC_JOB_MAX_TIMEOUT_DAYS} From 7a443b2901c5e593cc07182735ab27be4b903f49 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 14:26:42 -0700 Subject: [PATCH 23/32] reset to master --- .../source_file_secure/source.py | 4 +- .../source-postgres-strict-encrypt/Dockerfile | 2 +- .../connectors/source-postgres/Dockerfile | 2 +- .../relationaldb/StateDecoratingIterator.java | 43 ++++--- .../StateDecoratingIteratorTest.java | 112 ++++++++++++++++-- .../source-sendgrid/unit_tests/unit_test.py | 24 +++- 6 files changed, 154 insertions(+), 33 deletions(-) diff --git a/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py b/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py index db36e11d20533..52fcff45f431a 100644 --- a/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py +++ b/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py @@ -36,11 +36,11 @@ class URLFileSecure(ParentURLFile): This connector shouldn't work with local files. """ - def __init__(self, url: str, provider: dict): + def __init__(self, url: str, provider: dict, binary=None, encoding=None): storage_name = provider["storage"].lower() if url.startswith("file://") or storage_name == LOCAL_STORAGE_NAME: raise RuntimeError("the local file storage is not supported by this connector.") - super().__init__(url, provider) + super().__init__(url, provider, binary, encoding) class SourceFileSecure(ParentSourceFile): diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 5cde4f85fd45a..51f5cce85e189 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 9dfb9767a391a..9dae6b7cff80a 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java index d2880e26a3cdd..12370d9468b6d 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java @@ -13,6 +13,7 @@ import io.airbyte.protocol.models.AirbyteStateMessage; import io.airbyte.protocol.models.JsonSchemaPrimitive; import java.util.Iterator; +import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,10 +28,16 @@ public class StateDecoratingIterator extends AbstractIterator im private final JsonSchemaPrimitive cursorType; private final int stateEmissionFrequency; + private final String initialCursor; private String maxCursor; - private AirbyteMessage intermediateStateMessage; private boolean hasEmittedFinalState; - private int recordCount; + + // The intermediateStateMessage is set to the latest state message. + // For every stateEmissionFrequency messages, emitIntermediateState is set to true and + // the latest intermediateStateMessage will be emitted. + private int totalRecordCount = 0; + private boolean emitIntermediateState = false; + private AirbyteMessage intermediateStateMessage = null; /** * @param stateEmissionFrequency If larger than 0, intermediate states will be emitted for every @@ -49,6 +56,7 @@ public StateDecoratingIterator(final Iterator messageIterator, this.pair = pair; this.cursorField = cursorField; this.cursorType = cursorType; + this.initialCursor = initialCursor; this.maxCursor = initialCursor; this.stateEmissionFrequency = stateEmissionFrequency; } @@ -60,36 +68,41 @@ private String getCursorCandidate(final AirbyteMessage message) { @Override protected AirbyteMessage computeNext() { - if (intermediateStateMessage != null) { - final AirbyteMessage message = intermediateStateMessage; - intermediateStateMessage = null; - return message; - } else if (messageIterator.hasNext()) { - recordCount++; + if (messageIterator.hasNext()) { + if (emitIntermediateState && intermediateStateMessage != null) { + final AirbyteMessage message = intermediateStateMessage; + intermediateStateMessage = null; + emitIntermediateState = false; + return message; + } + + totalRecordCount++; final AirbyteMessage message = messageIterator.next(); if (message.getRecord().getData().hasNonNull(cursorField)) { final String cursorCandidate = getCursorCandidate(message); if (IncrementalUtils.compareCursors(maxCursor, cursorCandidate, cursorType) < 0) { + if (stateEmissionFrequency > 0 && !Objects.equals(maxCursor, initialCursor) && messageIterator.hasNext()) { + // Only emit an intermediate state when it is not the first or last record message, + // because the last state message will be taken care of in a different branch. + intermediateStateMessage = createStateMessage(false); + } maxCursor = cursorCandidate; } } - if (stateEmissionFrequency > 0 && recordCount % stateEmissionFrequency == 0) { - // Mark the state as final in case this intermediate state happens to be the last one. - // This is not necessary, but avoid sending the final states twice and prevent any edge case. - final boolean isFinalState = !messageIterator.hasNext(); - intermediateStateMessage = emitStateMessage(isFinalState); + if (stateEmissionFrequency > 0 && totalRecordCount % stateEmissionFrequency == 0) { + emitIntermediateState = true; } return message; } else if (!hasEmittedFinalState) { - return emitStateMessage(true); + return createStateMessage(true); } else { return endOfData(); } } - public AirbyteMessage emitStateMessage(final boolean isFinalState) { + public AirbyteMessage createStateMessage(final boolean isFinalState) { final AirbyteStateMessage stateMessage = stateManager.updateAndEmit(pair, maxCursor); LOGGER.info("State Report: stream name: {}, original cursor field: {}, original cursor value {}, cursor field: {}, new cursor value: {}", pair, diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java index 8f16a7d5a11ff..474d553d3a1d4 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java @@ -45,10 +45,18 @@ class StateDecoratingIteratorTest { private static final AirbyteMessage RECORD_MESSAGE_2 = createRecordMessage(RECORD_VALUE_2); private static final AirbyteMessage STATE_MESSAGE_2 = createStateMessage(RECORD_VALUE_2); - private static final String RECORD_VALUE_3 = "xyz"; + private static final String RECORD_VALUE_3 = "ghi"; private static final AirbyteMessage RECORD_MESSAGE_3 = createRecordMessage(RECORD_VALUE_3); private static final AirbyteMessage STATE_MESSAGE_3 = createStateMessage(RECORD_VALUE_3); + private static final String RECORD_VALUE_4 = "jkl"; + private static final AirbyteMessage RECORD_MESSAGE_4 = createRecordMessage(RECORD_VALUE_4); + private static final AirbyteMessage STATE_MESSAGE_4 = createStateMessage(RECORD_VALUE_4); + + private static final String RECORD_VALUE_5 = "xyz"; + private static final AirbyteMessage RECORD_MESSAGE_5 = createRecordMessage(RECORD_VALUE_5); + private static final AirbyteMessage STATE_MESSAGE_5 = createStateMessage(RECORD_VALUE_5); + private static AirbyteMessage createRecordMessage(final String recordValue) { return new AirbyteMessage() .withType(Type.RECORD) @@ -73,6 +81,8 @@ void setup() { when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_1)).thenReturn(STATE_MESSAGE_1.getState()); when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_2)).thenReturn(STATE_MESSAGE_2.getState()); when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_3)).thenReturn(STATE_MESSAGE_3.getState()); + when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_4)).thenReturn(STATE_MESSAGE_4.getState()); + when(stateManager.updateAndEmit(NAME_NAMESPACE_PAIR, RECORD_VALUE_5)).thenReturn(STATE_MESSAGE_5.getState()); when(stateManager.getOriginalCursorField(NAME_NAMESPACE_PAIR)).thenReturn(Optional.empty()); when(stateManager.getOriginalCursor(NAME_NAMESPACE_PAIR)).thenReturn(Optional.empty()); @@ -106,13 +116,13 @@ void testWithInitialCursor() { stateManager, NAME_NAMESPACE_PAIR, UUID_FIELD_NAME, - RECORD_VALUE_3, + RECORD_VALUE_5, JsonSchemaPrimitive.STRING, 0); assertEquals(RECORD_MESSAGE_1, iterator.next()); assertEquals(RECORD_MESSAGE_2, iterator.next()); - assertEquals(STATE_MESSAGE_3, iterator.next()); + assertEquals(STATE_MESSAGE_5, iterator.next()); assertFalse(iterator.hasNext()); } @@ -179,8 +189,8 @@ void testUnicodeNull() { @Test @DisplayName("When initial cursor is null, and emit state for every record") - void testStateEmission1() { - messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_3); + void testStateEmissionFrequency1() { + messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_3, RECORD_MESSAGE_4, RECORD_MESSAGE_5); final StateDecoratingIterator iterator1 = new StateDecoratingIterator( messageIterator, stateManager, @@ -191,19 +201,27 @@ void testStateEmission1() { 1); assertEquals(RECORD_MESSAGE_1, iterator1.next()); - assertEquals(STATE_MESSAGE_1, iterator1.next()); + // should emit state 1, but it is unclear whether there will be more + // records with the same cursor value, so no state is ready for emission assertEquals(RECORD_MESSAGE_2, iterator1.next()); - assertEquals(STATE_MESSAGE_2, iterator1.next()); + // emit state 1 because it is the latest state ready for emission + assertEquals(STATE_MESSAGE_1, iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); - // final state message should only be emitted once + assertEquals(STATE_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_4, iterator1.next()); assertEquals(STATE_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_5, iterator1.next()); + // state 4 is not emitted because there is no more record and only + // the final state should be emitted at this point; also the final + // state should only be emitted once + assertEquals(STATE_MESSAGE_5, iterator1.next()); assertFalse(iterator1.hasNext()); } @Test @DisplayName("When initial cursor is null, and emit state for every 2 records") - void testStateEmission2() { - messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_3); + void testStateEmissionFrequency2() { + messageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_3, RECORD_MESSAGE_4, RECORD_MESSAGE_5); final StateDecoratingIterator iterator1 = new StateDecoratingIterator( messageIterator, stateManager, @@ -215,16 +233,74 @@ void testStateEmission2() { assertEquals(RECORD_MESSAGE_1, iterator1.next()); assertEquals(RECORD_MESSAGE_2, iterator1.next()); - assertEquals(STATE_MESSAGE_2, iterator1.next()); + // emit state 1 because it is the latest state ready for emission + assertEquals(STATE_MESSAGE_1, iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_4, iterator1.next()); + // emit state 3 because it is the latest state ready for emission assertEquals(STATE_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_5, iterator1.next()); + assertEquals(STATE_MESSAGE_5, iterator1.next()); assertFalse(iterator1.hasNext()); } @Test @DisplayName("When initial cursor is not null") - void testStateEmission3() { - messageIterator = MoreIterators.of(RECORD_MESSAGE_2, RECORD_MESSAGE_3); + void testStateEmissionWhenInitialCursorIsNotNull() { + messageIterator = MoreIterators.of(RECORD_MESSAGE_2, RECORD_MESSAGE_3, RECORD_MESSAGE_4, RECORD_MESSAGE_5); + final StateDecoratingIterator iterator1 = new StateDecoratingIterator( + messageIterator, + stateManager, + NAME_NAMESPACE_PAIR, + UUID_FIELD_NAME, + RECORD_VALUE_1, + JsonSchemaPrimitive.STRING, + 1); + + assertEquals(RECORD_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(STATE_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_4, iterator1.next()); + assertEquals(STATE_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_5, iterator1.next()); + assertEquals(STATE_MESSAGE_5, iterator1.next()); + assertFalse(iterator1.hasNext()); + } + + /** + * Incremental syncs will sort the table with the cursor field, and emit the max cursor for every N + * records. The purpose is to emit the states frequently, so that if any transient failure occurs + * during a long sync, the next run does not need to start from the beginning, but can resume from + * the last successful intermediate state committed on the destination. The next run will start with + * `cursorField > cursor`. However, it is possible that there are multiple records with the same + * cursor value. If the intermediate state is emitted before all these records have been synced to + * the destination, some of these records may be lost. + *

+ * Here is an example: + * + * 

+   * | Record ID | Cursor Field | Other Field | Note                          |
+   * | --------- | ------------ | ----------- | ----------------------------- |
+   * | 1         | F1=16        | F2="abc"    |                               |
+   * | 2         | F1=16        | F2="def"    | <- state emission and failure |
+   * | 3         | F1=16        | F2="ghi"    |                               |
+   *
+ * + * If the intermediate state is emitted for record 2 and the sync fails immediately such that the + * cursor value `16` is committed, but only record 1 and 2 are actually synced, the next run will + * start with `F1 > 16` and skip record 3. + *

+ * So intermediate state emission should only happen when all records with the same cursor value has + * been synced to destination. Reference: https://github.com/airbytehq/airbyte/issues/15427 + */ + @Test + @DisplayName("When there are multiple records with the same cursor value") + void testStateEmissionForRecordsSharingSameCursorValue() { + messageIterator = MoreIterators.of( + RECORD_MESSAGE_2, RECORD_MESSAGE_2, + RECORD_MESSAGE_3, RECORD_MESSAGE_3, RECORD_MESSAGE_3, + RECORD_MESSAGE_4, + RECORD_MESSAGE_5, RECORD_MESSAGE_5); final StateDecoratingIterator iterator1 = new StateDecoratingIterator( messageIterator, stateManager, @@ -235,9 +311,19 @@ void testStateEmission3() { 1); assertEquals(RECORD_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_2, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + // state 2 is the latest state ready for emission because + // all records with the same cursor value have been emitted assertEquals(STATE_MESSAGE_2, iterator1.next()); assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_4, iterator1.next()); assertEquals(STATE_MESSAGE_3, iterator1.next()); + assertEquals(RECORD_MESSAGE_5, iterator1.next()); + assertEquals(STATE_MESSAGE_4, iterator1.next()); + assertEquals(RECORD_MESSAGE_5, iterator1.next()); + assertEquals(STATE_MESSAGE_5, iterator1.next()); assertFalse(iterator1.hasNext()); } diff --git a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py index 8f50befe1e23f..c5f8e71ff340b 100644 --- a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py @@ -2,13 +2,17 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import unittest from unittest.mock import MagicMock +import pendulum import pytest import requests from airbyte_cdk.logger import AirbyteLogger from source_sendgrid.source import SourceSendgrid -from source_sendgrid.streams import SendgridStream +from source_sendgrid.streams import Messages, SendgridStream + +FAKE_NOW = pendulum.DateTime(2022, 1, 1, tzinfo=pendulum.timezone("utc")) @pytest.fixture(name="sendgrid_stream") @@ -19,6 +23,13 @@ def sendgrid_stream_fixture(mocker) -> SendgridStream: return SendgridStream() # type: ignore +@pytest.fixture() +def mock_pendulum_now(monkeypatch): + pendulum_mock = unittest.mock.MagicMock(wraps=pendulum.now) + pendulum_mock.return_value = FAKE_NOW + monkeypatch.setattr(pendulum, "now", pendulum_mock) + + def test_parse_response_gracefully_handles_nulls(mocker, sendgrid_stream: SendgridStream): response = requests.Response() mocker.patch.object(response, "json", return_value=None) @@ -30,3 +41,14 @@ def test_source_wrong_credentials(): source = SourceSendgrid() status, error = source.check_connection(logger=AirbyteLogger(), config={"apikey": "wrong.api.key123"}) assert not status + + +def test_messages_stream_request_params(mock_pendulum_now): + start_time = 1558359837 + stream = Messages(start_time) + state = {"last_event_time": 1558359000} + request_params = stream.request_params(state) + assert ( + request_params + == "query=last_event_time%20BETWEEN%20TIMESTAMP%20%222019-05-20T06%3A30%3A00Z%22%20AND%20TIMESTAMP%20%222021-12-31T16%3A00%3A00Z%22&limit=1000" + ) From 182e4d8b0638228a889282ed61c16474308177d1 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 14:27:09 -0700 Subject: [PATCH 24/32] reset to master --- docs/integrations/sources/hubplanner.md | 42 +++++++++++++++++++++++++ docs/integrations/sources/postgres.md | 6 ++-- kube/overlays/dev/.env | 6 ++++ kube/overlays/stable/.env | 5 +++ kube/resources/worker.yaml | 25 +++++++++++++++ 5 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 docs/integrations/sources/hubplanner.md diff --git a/docs/integrations/sources/hubplanner.md b/docs/integrations/sources/hubplanner.md new file mode 100644 index 0000000000000..916e7dbd8fa35 --- /dev/null +++ b/docs/integrations/sources/hubplanner.md @@ -0,0 +1,42 @@ +# Hubplanner + +Hubplanner is a tool to plan, schedule, report and manage your entire team. + +## Prerequisites +* Create the API Key to access your data in Hubplanner. + +## Airbyte OSS +* API Key + +## Airbyte Cloud +* Comming Soon. + + +## Setup guide +### For Airbyte OSS: + +1. Access https://.hubplanner.com/settings#api or access the panel in left side Integrations/Hub Planner API +2. Click in Generate Key + +## Supported sync modes + +The Okta source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + - Full Refresh + +## Supported Streams + +- [Billing Rates](https://github.com/hubplanner/API/blob/master/Sections/billingrate.md) +- [Bookings](https://github.com/hubplanner/API/blob/master/Sections/bookings.md) +- [Clients](https://github.com/hubplanner/API/blob/master/Sections/clients.md) +- [Events](https://github.com/hubplanner/API/blob/master/Sections/events.md) +- [Holidays](https://github.com/hubplanner/API/blob/master/Sections/holidays.md) +- [Projects](https://github.com/hubplanner/API/blob/master/Sections/project.md) +- [Resources](https://github.com/hubplanner/API/blob/master/Sections/resource.md) + + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------| + +| 0.1.0 | 2021-08-10 | [12145](https://github.com/airbytehq/airbyte/pull/12145) | Initial Release | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 1f66efb72ac31..58e8a83b0b2c4 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -371,18 +371,20 @@ Possible solutions include: | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------------------| +| 1.0.1 | 2022-08-10 | [15496](https://github.com/airbytehq/airbyte/pull/15496) | Fix state emission in incremental sync | +| | 2022-08-10 | [15481](https://github.com/airbytehq/airbyte/pull/15481) | Fix data handling from WAL logs in CDC mode | | 1.0.0 | 2022-08-05 | [15380](https://github.com/airbytehq/airbyte/pull/15380) | Change connector label to generally_available | | 0.4.44 | 2022-08-05 | [15342](https://github.com/airbytehq/airbyte/pull/15342) | Adjust titles and descriptions in spec.json | | 0.4.43 | 2022-08-03 | [15226](https://github.com/airbytehq/airbyte/pull/15226) | Make connectionTimeoutMs configurable through JDBC url parameters | | 0.4.42 | 2022-08-03 | [15273](https://github.com/airbytehq/airbyte/pull/15273) | Fix a bug in `0.4.36` and correctly parse the CDC initial record waiting time | | 0.4.41 | 2022-08-03 | [15077](https://github.com/airbytehq/airbyte/pull/15077) | Sync data from beginning if the LSN is no longer valid in CDC | -| | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently | +| | 2022-08-03 | [14903](https://github.com/airbytehq/airbyte/pull/14903) | Emit state messages more frequently (⛔ this version has a bug; use `1.0.1` instead) | | 0.4.40 | 2022-08-03 | [15187](https://github.com/airbytehq/airbyte/pull/15187) | Add support for BCE dates/timestamps | | | 2022-08-03 | [14534](https://github.com/airbytehq/airbyte/pull/14534) | Align regular and CDC integration tests and data mappers | | 0.4.39 | 2022-08-02 | [14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | | 0.4.38 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | | 0.4.37 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead) | +| 0.4.36 | 2022-07-21 | [14451](https://github.com/airbytehq/airbyte/pull/14451) | Make initial CDC waiting time configurable (⛔ this version has a bug and will not work; use `0.4.42` instead) | | 0.4.35 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | | 0.4.34 | 2022-07-17 | [13840](https://github.com/airbytehq/airbyte/pull/13840) | Added the ability to connect using different SSL modes and SSL certificates. | | 0.4.33 | 2022-07-14 | [14586](https://github.com/airbytehq/airbyte/pull/14586) | Validate source JDBC url parameters | diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env index d622254798359..230efad8e4ece 100644 --- a/kube/overlays/dev/.env +++ b/kube/overlays/dev/.env @@ -56,6 +56,11 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= + # Worker pod tolerations, annotations and node selectors JOB_KUBE_TOLERATIONS= JOB_KUBE_ANNOTATIONS= @@ -66,6 +71,7 @@ JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY= # Launch a separate pod to orchestrate sync steps CONTAINER_ORCHESTRATOR_ENABLED=true +CONTAINER_ORCHESTRATOR_IMAGE= # Open Telemetry Configuration METRIC_CLIENT= diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index b48947558c899..e1b01c39f2704 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -56,6 +56,11 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= + # Worker pod tolerations, annotations and node selectors JOB_KUBE_TOLERATIONS= JOB_KUBE_ANNOTATIONS= diff --git a/kube/resources/worker.yaml b/kube/resources/worker.yaml index f5906abd1914a..02ceacb266e26 100644 --- a/kube/resources/worker.yaml +++ b/kube/resources/worker.yaml @@ -119,6 +119,26 @@ spec: configMapKeyRef: name: airbyte-env key: JOB_MAIN_CONTAINER_MEMORY_LIMIT + - name: NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST + - name: NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT + - name: NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: airbyte-env + key: NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST + - name: NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: airbyte-env + key: NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT - name: S3_LOG_BUCKET valueFrom: configMapKeyRef: @@ -210,6 +230,11 @@ spec: configMapKeyRef: name: airbyte-env key: CONTAINER_ORCHESTRATOR_ENABLED + - name: CONTAINER_ORCHESTRATOR_IMAGE + valueFrom: + configMapKeyRef: + name: airbyte-env + key: CONTAINER_ORCHESTRATOR_IMAGE - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION valueFrom: configMapKeyRef: From 0a9f825c783c0bb91599da38794aa5402969caa6 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 14:27:30 -0700 Subject: [PATCH 25/32] enable backward compatibility test --- .../connectors/source-sentry/acceptance-test-config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml index 2f9cf7f1a9ab6..6e27bef75cf46 100644 --- a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml @@ -2,8 +2,6 @@ connector_image: airbyte/source-sentry:dev tests: spec: - spec_path: "source_sentry/spec.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.1" connection: - config_path: "secrets/config.json" status: "succeed" From 4f16cb970949d83878f99633df218f572a36e403 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 14:27:53 -0700 Subject: [PATCH 26/32] bump cdk version --- airbyte-integrations/connectors/source-sentry/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-sentry/setup.py b/airbyte-integrations/connectors/source-sentry/setup.py index 2ebf8d6a31cb2..2c6a8538a5f03 100644 --- a/airbyte-integrations/connectors/source-sentry/setup.py +++ b/airbyte-integrations/connectors/source-sentry/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1.72", + "airbyte-cdk~=0.1.74", ] TEST_REQUIREMENTS = [ From 5338da3b1ed0b6c6b23804cb882cc9a1bd3e9afa Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 14:32:06 -0700 Subject: [PATCH 27/32] reset --- .env | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.env b/.env index a8f8f95d64828..8efe0cda076ae 100644 --- a/.env +++ b/.env @@ -67,6 +67,10 @@ JOB_MAIN_CONTAINER_CPU_LIMIT= JOB_MAIN_CONTAINER_MEMORY_REQUEST= JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT= +NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST= ### LOGGING/MONITORING/TRACKING ### TRACKING_STRATEGY=segment From a2a0db88a0dd531072b83183e1f988224f1b645d Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 15:05:55 -0700 Subject: [PATCH 28/32] Update airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml Co-authored-by: Sherif A. Nada --- .../connectors/source-sentry/source_sentry/sentry.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml index 567e3aad8ced1..8fa1dc20fbbd2 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml @@ -37,6 +37,7 @@ definitions: streams: - type: DeclarativeStream $options: + # https://docs.sentry.io/api/events/list-a-projects-events/ name: "events" primary_key: "id" schema_loader: From 37321d605f85e0a2ad96af723a65931246eeb2a5 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 15:08:14 -0700 Subject: [PATCH 29/32] Use paginator --- .../connectors/source-sentry/source_sentry/sentry.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml index 8fa1dc20fbbd2..737665e0aaee5 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml @@ -72,7 +72,7 @@ streams: statsPeriod: "" query: "" paginator: - type: NoPagination + $ref: "*ref(definitions.paginator)" - type: DeclarativeStream $options: name: "projects" @@ -87,7 +87,7 @@ streams: $ref: "*ref(definitions.requester)" path: "projects/" paginator: - type: NoPagination + $ref: "*ref(definitions.paginator)" - type: DeclarativeStream $options: name: "project_detail" From 5a9bb1ed67f4ee04f83e6b4a24a6a7b12871fd8b Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 15:21:04 -0700 Subject: [PATCH 30/32] fix pagination --- .../connectors/source-sentry/source_sentry/sentry.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml index 737665e0aaee5..7a69b16a37b77 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/sentry.yaml @@ -24,7 +24,8 @@ definitions: inject_into: "request_parameter" field_name: "" page_token_option: - inject_into: "path" + inject_into: "request_parameter" + field_name: "cursor" pagination_strategy: type: "CursorPagination" cursor_value: "{{ headers.link.next.cursor }}" From 8e062cb4a13e47efe07fa0f99e7eec2b966f13ed Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Thu, 11 Aug 2022 18:53:59 -0700 Subject: [PATCH 31/32] bump version --- airbyte-integrations/connectors/source-sentry/Dockerfile | 2 +- docs/integrations/sources/sentry.md | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-sentry/Dockerfile b/airbyte-integrations/connectors/source-sentry/Dockerfile index 12c54ceaa2adb..6fc810db4ee94 100644 --- a/airbyte-integrations/connectors/source-sentry/Dockerfile +++ b/airbyte-integrations/connectors/source-sentry/Dockerfile @@ -34,5 +34,5 @@ COPY source_sentry ./source_sentry ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-sentry diff --git a/docs/integrations/sources/sentry.md b/docs/integrations/sources/sentry.md index 055c1cdcc56e4..1e4c696f2b76d 100644 --- a/docs/integrations/sources/sentry.md +++ b/docs/integrations/sources/sentry.md @@ -44,7 +44,8 @@ You can find or create authentication tokens within [Sentry](https://sentry.io/s ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.1 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | -| 0.1.0 | 2021-10-12 | [6975](https://github.com/airbytehq/airbyte/pull/6975) | New Source: Sentry | +| Version | Date | Pull Request | Subject | +|:--------| :--- | :--- |:--------------------------------------------------| +| 0.1.2 | 2021-12-28 | [15345](https://github.com/airbytehq/airbyte/pull/15345) | Migrate to config-based framework | +| 0.1.1 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | +| 0.1.0 | 2021-10-12 | [6975](https://github.com/airbytehq/airbyte/pull/6975) | New Source: Sentry | From f4a97235e9f1dcdf10d2d5731e46ddf7758ad5d1 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 12 Aug 2022 02:17:33 +0000 Subject: [PATCH 32/32] auto-bump connector version [ci skip] --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 107f98e1ffd7d..4da67aefa27a0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1103,7 +1103,7 @@ - sourceDefinitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781 name: Sentry dockerRepository: airbyte/source-sentry - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/sentry icon: sentry.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 2eb3d36376ad9..5adb3fcdd0cdd 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -10694,7 +10694,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-sentry:0.1.1" +- dockerImage: "airbyte/source-sentry:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/sentry" connectionSpecification: @@ -10705,7 +10705,7 @@ - "auth_token" - "organization" - "project" - additionalProperties: false + additionalProperties: true properties: auth_token: type: "string"

diff --git a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldRow.tsx b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldRow.tsx index 1c77e9a7281af..3f8d715c59065 100644 --- a/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldRow.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogDiffModal/components/FieldRow.tsx @@ -28,7 +28,7 @@ export const FieldRow: React.FC = ({ transform }) => { [styles.mod]: diffType === "update", }); - const contentStyle = classnames(styles.content, { + const contentStyle = classnames(styles.content, styles.cell, { [styles.add]: diffType === "add", [styles.remove]: diffType === "remove", [styles.update]: diffType === "update", @@ -50,16 +50,16 @@ export const FieldRow: React.FC = ({ transform }) => { )} - +
{fieldName} -
+ +
{oldType && newType && ( {oldType} {newType} )} -
{namespace}{itemName}{itemName} {diffVerb === "removed" && syncMode && }