Skip to content

Commit bae5b02

Browse files
vitaliizazmicvitaliizazmic
authored
🎉 Source Microsoft Teams: migrate to the CDK
* Source Microsoft Teams #8315 - migrate to the CDK * Source Microsoft Teams #8315 - acceptance tests * Source Google Sheets #8313 - removing configured_catalog for SAT according to PR review * Source Google Sheets #8313 - bump version and update docs * Source Microsoft Teams #8315 - update source spec
1 parent 0eaaa1f commit bae5b02

File tree

18 files changed

+132
-41
lines changed

18 files changed

+132
-41
lines changed

‎airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eaf50f04-21dd-4620-913b-2a83f5635227.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"sourceDefinitionId": "eaf50f04-21dd-4620-913b-2a83f5635227",
33
"name": "Microsoft teams",
44
"dockerRepository": "airbyte/source-microsoft-teams",
5-
"dockerImageTag": "0.2.2",
5+
"dockerImageTag": "0.2.3",
66
"documentationUrl": "https://docs.airbyte.io/integrations/sources/microsoft-teams",
77
"icon": "microsoft-teams.svg"
88
}

‎airbyte-config/init/src/main/resources/seed/source_definitions.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -386,7 +386,7 @@
386386
- name: Microsoft teams
387387
sourceDefinitionId: eaf50f04-21dd-4620-913b-2a83f5635227
388388
dockerRepository: airbyte/source-microsoft-teams
389-
dockerImageTag: 0.2.2
389+
dockerImageTag: 0.2.3
390390
documentationUrl: https://docs.airbyte.io/integrations/sources/microsoft-teams
391391
icon: microsoft-teams.svg
392392
sourceType: api

‎airbyte-config/init/src/main/resources/seed/source_specs.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3728,7 +3728,7 @@
37283728
supportsNormalization: false
37293729
supportsDBT: false
37303730
supported_destination_sync_modes: []
3731-
- dockerImage: "airbyte/source-microsoft-teams:0.2.2"
3731+
- dockerImage: "airbyte/source-microsoft-teams:0.2.3"
37323732
spec:
37333733
documentationUrl: "https://docs.airbyte.io/integrations/sources/microsoft-teams"
37343734
connectionSpecification:

‎airbyte-integrations/connectors/source-microsoft-teams/.dockerignore

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
*
22
!Dockerfile
3-
!Dockerfile.test
3+
!main.py
44
!source_microsoft_teams
55
!setup.py
6-
!sample_files
7-
!integration_tests
86
!secrets

‎airbyte-integrations/connectors/source-microsoft-teams/Dockerfile

+31-11
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,38 @@
1-
FROM airbyte/integration-base-python:0.1.1
1+
FROM python:3.7.11-alpine3.14 as base
22

3-
# Bash is installed for more convenient debugging.
4-
RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/*
3+
# build and load all requirements
4+
FROM base as builder
5+
WORKDIR /airbyte/integration_code
6+
7+
# upgrade pip to the latest version
8+
RUN apk --no-cache upgrade \
9+
&& pip install --upgrade pip \
10+
&& apk --no-cache add tzdata build-base libffi-dev
511

6-
ENV CODE_PATH="source_microsoft_teams"
7-
ENV AIRBYTE_IMPL_MODULE="source_microsoft_teams"
8-
ENV AIRBYTE_IMPL_PATH="SourceMicrosoftTeams"
912

10-
WORKDIR /airbyte/integration_code
11-
COPY $CODE_PATH ./$CODE_PATH
1213
COPY setup.py ./
13-
RUN pip install .
14+
# install necessary packages to a temporary folder
15+
RUN pip install --prefix=/install .
16+
17+
# build a clean environment
18+
FROM base
19+
WORKDIR /airbyte/integration_code
20+
21+
# copy all loaded and built libraries to a pure basic image
22+
COPY --from=builder /install /usr/local
23+
# add default timezone settings
24+
COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
25+
RUN echo "Etc/UTC" > /etc/timezone
26+
27+
# bash is installed for more convenient debugging.
28+
RUN apk --no-cache add bash
29+
30+
# copy payload code only
31+
COPY main.py ./
32+
COPY source_microsoft_teams ./source_microsoft_teams
1433

15-
ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
34+
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
35+
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
1636

17-
LABEL io.airbyte.version=0.2.2
37+
LABEL io.airbyte.version=0.2.3
1838
LABEL io.airbyte.name=airbyte/source-microsoft-teams

‎airbyte-integrations/connectors/source-microsoft-teams/acceptance-test-config.yml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference)
2+
# for more information about how to configure these tests
3+
connector_image: airbyte/source-microsoft-teams:dev
4+
tests:
5+
spec:
6+
- spec_path: "source_microsoft_teams/spec.json"
7+
connection:
8+
- config_path: "secrets/config.json"
9+
status: "succeed"
10+
- config_path: "integration_tests/invalid_config.json"
11+
status: "exception"
12+
discovery:
13+
- config_path: "secrets/config.json"
14+
basic_read:
15+
- config_path: "secrets/config.json"
16+
empty_streams: []
17+
full_refresh:
18+
- config_path: "secrets/config.json"

‎airbyte-integrations/connectors/source-microsoft-teams/acceptance-test-docker.sh

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env sh
2+
3+
# Build latest connector image
4+
docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2)
5+
6+
# Pull latest acctest image
7+
docker pull airbyte/source-acceptance-test:latest
8+
9+
# Run
10+
docker run --rm -it \
11+
-v /var/run/docker.sock:/var/run/docker.sock \
12+
-v /tmp:/tmp \
13+
-v $(pwd):/test_input \
14+
airbyte/source-acceptance-test \
15+
--acceptance-test-config /test_input
16+

‎airbyte-integrations/connectors/source-microsoft-teams/build.gradle

+1-14
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,10 @@
11
plugins {
22
id 'airbyte-python'
33
id 'airbyte-docker'
4-
id 'airbyte-standard-source-test-file'
4+
id 'airbyte-source-acceptance-test'
55
}
66

77
airbytePython {
88
moduleDirectory 'source_microsoft_teams'
99
}
1010

11-
12-
airbyteStandardSourceTestFile {
13-
specPath = "source_microsoft_teams/spec.json"
14-
configPath = "secrets/config.json"
15-
configuredCatalogPath = "sample_files/configured_catalog.json"
16-
}
17-
18-
19-
20-
dependencies {
21-
implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs)
22-
implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
23-
}

‎airbyte-integrations/connectors/source-microsoft-teams/integration_tests/__init__.py

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#
2+
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
3+
#

‎airbyte-integrations/connectors/source-microsoft-teams/integration_tests/acceptance.py

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#
2+
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
3+
#
4+
5+
6+
import pytest
7+
8+
pytest_plugins = ("source_acceptance_test.plugin",)
9+
10+
11+
@pytest.fixture(scope="session", autouse=True)
12+
def connector_setup():
13+
"""This fixture is a placeholder for external resources that acceptance test might require."""
14+
yield
15+

‎airbyte-integrations/connectors/source-microsoft-teams/integration_tests/invalid_config.json

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"client_id": "wrong_client_id",
3+
"tenant_id": "wrong_tenant_id",
4+
"client_secret": "wrong_client_secret",
5+
"period": "D180"
6+
}

‎airbyte-integrations/connectors/source-microsoft-teams/integration_tests/sample_config.json

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"tenant_id": "<your tenant ID>",
3+
"client_id": "<your client ID>",
4+
"client_secret": "<your client secret>",
5+
"period": "D7"
6+
}

‎airbyte-integrations/connectors/source-microsoft-teams/main_dev.py ‎airbyte-integrations/connectors/source-microsoft-teams/main.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
import sys
77

8-
from base_python.entrypoint import launch
8+
from airbyte_cdk.entrypoint import launch
99
from source_microsoft_teams import SourceMicrosoftTeams
1010

1111
if __name__ == "__main__":

‎airbyte-integrations/connectors/source-microsoft-teams/requirements.txt

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
11
# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies.
2-
-e ../../bases/airbyte-protocol
3-
-e ../../bases/base-python
2+
-e ../../bases/source-acceptance-test
43
-e .

‎airbyte-integrations/connectors/source-microsoft-teams/setup.py

+18-2
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,28 @@
55

66
from setuptools import find_packages, setup
77

8+
MAIN_REQUIREMENTS = [
9+
"airbyte-cdk~=0.1",
10+
"requests",
11+
"msal==1.7.0",
12+
"backoff",
13+
]
14+
15+
TEST_REQUIREMENTS = [
16+
"pytest~=6.1",
17+
"pytest-mock~=3.6.1",
18+
"source-acceptance-test",
19+
]
20+
821
setup(
922
name="source_microsoft_teams",
1023
description="Source implementation for Microsoft Teams.",
1124
author="Airbyte",
1225
author_email="contact@airbyte.io",
1326
packages=find_packages(),
14-
install_requires=["airbyte-protocol", "base-python", "requests", "msal==1.7.0", "backoff", "pytest==6.1.2"],
15-
package_data={"": ["*.json", "schemas/*.json"]},
27+
install_requires=MAIN_REQUIREMENTS,
28+
package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]},
29+
extras_require={
30+
"tests": TEST_REQUIREMENTS,
31+
},
1632
)

‎airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/client.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@
1414
import backoff
1515
import msal
1616
import requests
17-
from airbyte_protocol import AirbyteStream
18-
from base_python import AirbyteLogger
17+
from airbyte_cdk.models.airbyte_protocol import AirbyteStream
18+
from airbyte_cdk.logger import AirbyteLogger
1919
from msal.exceptions import MsalServiceError
2020

2121
LOGGER = AirbyteLogger()
@@ -129,7 +129,7 @@ def get_streams(self):
129129
streams = []
130130
for schema, method in self.ENTITY_MAP.items():
131131
raw_schema = json.loads(pkgutil.get_data(self.__class__.__module__.split(".")[0], f"schemas/{schema}.json"))
132-
streams.append(AirbyteStream(name=schema, json_schema=raw_schema))
132+
streams.append(AirbyteStream(name=schema, json_schema=raw_schema, supported_sync_modes=["full_refresh"]))
133133
return streams
134134

135135
def get_users(self):

‎airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/source.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
from datetime import datetime
88
from typing import Dict, Generator
99

10-
from airbyte_protocol import AirbyteCatalog, AirbyteMessage, AirbyteRecordMessage, ConfiguredAirbyteCatalog, Type
11-
from base_python import AirbyteLogger, BaseSource
10+
from airbyte_cdk.logger import AirbyteLogger
11+
from airbyte_cdk.models.airbyte_protocol import AirbyteCatalog, AirbyteMessage, AirbyteRecordMessage, ConfiguredAirbyteCatalog, Type
12+
from airbyte_cdk.sources.deprecated.base_source import BaseSource
1213

1314
from .client import Client
1415

‎docs/integrations/sources/microsoft-teams.md

+6
Original file line numberDiff line numberDiff line change
@@ -155,3 +155,9 @@ This source requires **Application permissions**. Follow these [instructions](ht
155155

156156
Token acquiring implemented by [instantiate](https://docs.microsoft.com/en-us/azure/active-directory/develop/scenario-daemon-app-configuration?tabs=python#instantiate-the-msal-application) the confidential client application with a client secret and [calling](https://docs.microsoft.com/en-us/azure/active-directory/develop/scenario-daemon-acquire-token?tabs=python) AcquireTokenForClient from [Microsoft Authentication Library \(MSAL\) for Python](https://github.com/AzureAD/microsoft-authentication-library-for-python)
157157

158+
## CHANGELOG
159+
160+
| Version | Date | Pull Request | Subject |
161+
|:--------|:-----------| :--- | :--- |
162+
| 0.2.3 | 2021-12-06 | [8469](https://github.com/airbytehq/airbyte/pull/8469) | Migrate to the CDK |
163+

0 commit comments

Comments
 (0)