diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index f1309c70528f0..04ae58b281f8d 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -73,6 +73,7 @@ | OneSignal | [![source-onesignal](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-onesignal%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-onesignal) | | OpenWeather | [![source-openweather](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-openweather%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-openweather) | | Oracle DB | [![source-oracle](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-oracle%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-oracle) | +| Orbit | [![source-orbit](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-orbit%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-orbit) | | Paypal Transaction | [![paypal-transaction](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-paypal-transaction%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-paypal-transaction) | | Paystack | [![source-paystack](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-paystack%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-paystack) | | PersistIq | [![source-persistiq](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-persistiq%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-persistiq/) | diff --git a/airbyte-integrations/connectors/source-orbit/.dockerignore b/airbyte-integrations/connectors/source-orbit/.dockerignore new file mode 100644 index 0000000000000..694d552c2125c --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_orbit +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-orbit/Dockerfile b/airbyte-integrations/connectors/source-orbit/Dockerfile new file mode 100644 index 0000000000000..f25b598bad68b --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_orbit ./source_orbit + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-orbit diff --git a/airbyte-integrations/connectors/source-orbit/README.md b/airbyte-integrations/connectors/source-orbit/README.md new file mode 100644 index 0000000000000..bb71b2d2e912e --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/README.md @@ -0,0 +1,132 @@ +# Orbit Source + +This is the repository for the Orbit source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/orbit). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-orbit:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/orbit) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_orbit/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source orbit test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-orbit:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-orbit:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-orbit:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-orbit:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-orbit:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-orbit:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-orbit:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-orbit:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-orbit/acceptance-test-config.yml b/airbyte-integrations/connectors/source-orbit/acceptance-test-config.yml new file mode 100644 index 0000000000000..a00444b24a6d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-orbit:dev +tests: + spec: + - spec_path: "source_orbit/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-orbit/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-orbit/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-orbit/build.gradle b/airbyte-integrations/connectors/source-orbit/build.gradle new file mode 100644 index 0000000000000..198305c2ab138 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_orbit' +} diff --git a/airbyte-integrations/connectors/source-orbit/integration_tests/__init__.py b/airbyte-integrations/connectors/source-orbit/integration_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-orbit/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-orbit/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..52b0f2c2118f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-orbit/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-orbit/integration_tests/acceptance.py new file mode 100644 index 0000000000000..1302b2f57e10e --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-orbit/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-orbit/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..a0d0091af9f8e --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/integration_tests/configured_catalog.json @@ -0,0 +1,44 @@ +{ + "streams": [ + { + "stream": { + "name": "members", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_token": { + "type": "string" + }, + "workspace": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "workspace", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_token": { + "type": "string" + }, + "workspace": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-orbit/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-orbit/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..7e719f4a39a12 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/integration_tests/invalid_config.json @@ -0,0 +1 @@ +{ "api_token": "obw_token", "workspace": "airbyte" } diff --git a/airbyte-integrations/connectors/source-orbit/main.py b/airbyte-integrations/connectors/source-orbit/main.py new file mode 100644 index 0000000000000..a7b3c933efac5 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_orbit import SourceOrbit + +if __name__ == "__main__": + source = SourceOrbit() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-orbit/requirements.txt b/airbyte-integrations/connectors/source-orbit/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-orbit/setup.py b/airbyte-integrations/connectors/source-orbit/setup.py new file mode 100644 index 0000000000000..00a30d7899550 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1.56", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_orbit", + description="Source implementation for Orbit.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/__init__.py b/airbyte-integrations/connectors/source-orbit/source_orbit/__init__.py new file mode 100644 index 0000000000000..4888354eaa19f --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceOrbit + +__all__ = ["SourceOrbit"] diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/members.json b/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/members.json new file mode 100644 index 0000000000000..3123a78cd23e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/members.json @@ -0,0 +1,155 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "fake": { + "type": "string" + }, + "type": { + "type": "string" + }, + "attributes": { + "type": "object", + "properties": { + "activities_count": { + "type": "integer" + }, + "activities_score": { + "type": "integer" + }, + "avatar_url": { + "type": ["null", "string"] + }, + "bio": { + "type": ["null", "string"] + }, + "birthday": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "deleted_at": { + "type": ["null", "string"] + }, + "first_activity_occurred_at": { + "type": ["null", "string"] + }, + "last_activity_occurred_at": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "pronouns": { + "type": ["null", "string"] + }, + "reach": { + "type": ["null", "integer"] + }, + "shipping_address": { + "type": ["null", "string"] + }, + "slug": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "tag_list": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "tags": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "teammate": { + "type": "boolean" + }, + "tshirt": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "merged_at": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "orbit_url": { + "type": ["null", "string"] + }, + "created": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "orbit_level": { + "type": ["null", "integer"] + }, + "love": { + "type": ["null", "string"] + }, + "twitter": { + "type": ["null", "string"] + }, + "github": { + "type": ["null", "string"] + }, + "discourse": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "devto": { + "type": ["null", "string"] + }, + "linkedin": { + "type": ["null", "string"] + }, + "discord": { + "type": ["null", "string"] + }, + "github_followers": { + "type": ["null", "integer"] + }, + "twitter_followers": { + "type": ["null", "integer"] + }, + "topics": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "languages": { + "type": ["null", "array"], + "items": { + "type": "string" + } + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/workspace.json b/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/workspace.json new file mode 100644 index 0000000000000..11261e17f0df1 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/workspace.json @@ -0,0 +1,38 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "attributes": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "updated_at": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "members_count": { + "type": "integer" + }, + "activities_count": { + "type": "integer" + }, + "tags": { + "type": "object" + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/workspace_old.json b/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/workspace_old.json new file mode 100644 index 0000000000000..f0a81fda814e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/workspace_old.json @@ -0,0 +1,568 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "attributes": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "updated_at": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "members_count": { + "type": "integer" + }, + "activities_count": { + "type": "integer" + }, + "tags": { + "type": "object" + } + }, + }, + "relationships": { + "type": "object", + "properties": { + "last_member": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + } + }, + }, + "last_activity": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + } + }, + }, + "repositories": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + } + ] + } + }, + } + }, + } + }, + }, + "included": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "attributes": { + "type": "object", + "properties": { + "activities_count": { + "type": "integer" + }, + "activities_score": { + "type": "integer" + }, + "avatar_url": { + "type": "string" + }, + "bio": { + "type": "string" + }, + "birthday": { + "type": "null" + }, + "company": { + "type": "null" + }, + "title": { + "type": "null" + }, + "created_at": { + "type": "string" + }, + "deleted_at": { + "type": "null" + }, + "first_activity_occurred_at": { + "type": "string" + }, + "last_activity_occurred_at": { + "type": "string" + }, + "location": { + "type": "string" + }, + "name": { + "type": "string" + }, + "pronouns": { + "type": "null" + }, + "reach": { + "type": "integer" + }, + "shipping_address": { + "type": "null" + }, + "slug": { + "type": "string" + }, + "source": { + "type": "string" + }, + "tag_list": { + "type": "array", + "items": {} + }, + "tags": { + "type": "array", + "items": {} + }, + "teammate": { + "type": "boolean" + }, + "tshirt": { + "type": "null" + }, + "updated_at": { + "type": "string" + }, + "merged_at": { + "type": "null" + }, + "url": { + "type": "string" + }, + "orbit_url": { + "type": "string" + }, + "created": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "orbit_level": { + "type": "integer" + }, + "love": { + "type": "string" + }, + "twitter": { + "type": "null" + }, + "github": { + "type": "string" + }, + "discourse": { + "type": "null" + }, + "email": { + "type": "null" + }, + "devto": { + "type": "null" + }, + "linkedin": { + "type": "null" + }, + "discord": { + "type": "null" + }, + "github_followers": { + "type": "integer" + }, + "twitter_followers": { + "type": "null" + }, + "topics": { + "type": "array", + "items": [ + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + } + ] + }, + "languages": { + "type": "array", + "items": [ + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + } + ] + } + }, + "required": [ + "activities_count", + "activities_score", + "avatar_url", + "bio", + "birthday", + "company", + "title", + "created_at", + "deleted_at", + "first_activity_occurred_at", + "last_activity_occurred_at", + "location", + "name", + "pronouns", + "reach", + "shipping_address", + "slug", + "source", + "tag_list", + "tags", + "teammate", + "tshirt", + "updated_at", + "merged_at", + "url", + "orbit_url", + "created", + "id", + "orbit_level", + "love", + "twitter", + "github", + "discourse", + "email", + "devto", + "linkedin", + "discord", + "github_followers", + "twitter_followers", + "topics", + "languages" + ] + }, + "relationships": { + "type": "object", + "properties": { + "identities": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": ["id", "type"] + } + ] + } + }, + "required": ["data"] + } + }, + "required": ["identities"] + } + }, + "required": ["id", "type", "attributes", "relationships"] + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "attributes": { + "type": "object", + "properties": { + "action": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "key": { + "type": "string" + }, + "occurred_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + }, + "type": { + "type": "string" + }, + "properties": { + "type": "object", + "properties": { + "github_repository": { + "type": "string" + }, + "github_organization": { + "type": "string" + }, + "github_pull_request": { + "type": "string" + } + }, + "required": [ + "github_repository", + "github_organization", + "github_pull_request" + ] + }, + "tags": { + "type": "array", + "items": [ + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + } + ] + }, + "orbit_url": { + "type": "string" + }, + "weight": { + "type": "string" + }, + "activity_link": { + "type": "string" + }, + "g_title": { + "type": "string" + }, + "g_number": { + "type": "integer" + }, + "g_html_url": { + "type": "string" + }, + "g_created_at": { + "type": "string" + }, + "is_pull_request": { + "type": "boolean" + }, + "g_merged": { + "type": "boolean" + }, + "g_merged_at": { + "type": "string" + }, + "g_merged_by": { + "type": "string" + } + }, + "required": [ + "action", + "created_at", + "key", + "occurred_at", + "updated_at", + "type", + "properties", + "tags", + "orbit_url", + "weight", + "activity_link", + "g_title", + "g_number", + "g_html_url", + "g_created_at", + "is_pull_request", + "g_merged", + "g_merged_at", + "g_merged_by" + ] + }, + "relationships": { + "type": "object", + "properties": { + "activity_type": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": ["id", "type"] + } + }, + "required": ["data"] + }, + "member": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": ["id", "type"] + } + }, + "required": ["data"] + }, + "repository": { + "type": "object", + "properties": { + "data": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": ["id", "type"] + } + }, + "required": ["data"] + } + }, + "required": ["activity_type", "member", "repository"] + } + }, + "required": ["id", "type", "attributes", "relationships"] + } + ] + } + }, + "required": ["data", "included"] +} diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/source.py b/airbyte-integrations/connectors/source-orbit/source_orbit/source.py new file mode 100644 index 0000000000000..280fff7d90675 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/source.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, List, Mapping, Tuple + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + +from .streams import Members, Workspace + + +# Source +class SourceOrbit(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + try: + workspace_stream = Workspace( + authenticator=TokenAuthenticator(token=config["api_token"]), + workspace=config["workspace"], + ) + next(workspace_stream.read_records(sync_mode=SyncMode.full_refresh)) + return True, None + except Exception as e: + return False, f"Please check that your API key and workspace name are entered correctly: {repr(e)}" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + + stream_kwargs = { + "authenticator": TokenAuthenticator(config["api_token"]), + "workspace": config["workspace"], + "start_date": config["start_date"], + } + + return [Members(**stream_kwargs), Workspace(**stream_kwargs)] diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/spec.yaml b/airbyte-integrations/connectors/source-orbit/source_orbit/spec.yaml new file mode 100644 index 0000000000000..8277b6d615390 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/spec.yaml @@ -0,0 +1,29 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/orbit +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Orbit Source Spec + type: object + required: + - api_token + - workspace + additionalProperties: false + properties: + api_token: + type: string + airbyte_secret: true + title: API Token + description: Authorizes you to work with Orbit workspaces associated with the token. + order: 0 + workspace: + type: string + title: Workspace + description: The unique name of the workspace that your API token is associated with. + order: 1 + start_date: + type: string + title: Start Date + description: >- + Date in the format 2022-06-26. Only load members whose last activities are after this date. + pattern: >- + ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + order: 2 diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/streams.py b/airbyte-integrations/connectors/source-orbit/source_orbit/streams.py new file mode 100644 index 0000000000000..5645af9536006 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/streams.py @@ -0,0 +1,96 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import urllib.parse +from abc import ABC +from typing import Any, Iterable, Mapping, MutableMapping, Optional + +import requests +from airbyte_cdk.sources.streams.http import HttpStream + + +class OrbitStream(HttpStream, ABC): + url_base = "https://app.orbit.love/api/v1/" + + def __init__(self, workspace: str, start_date: Optional[str] = None, **kwargs): + super().__init__(**kwargs) + self.workspace = workspace + self.start_date = start_date + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + data = response.json() + records = data["data"] + yield from records + + +class OrbitStreamPaginated(OrbitStream): + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, str]]: + decoded_response = response.json() + links = decoded_response.get("links") + if not links: + return None + + next = links.get("next") + if not next: + return None + + next_url = urllib.parse.urlparse(next) + return {str(k): str(v) for (k, v) in urllib.parse.parse_qsl(next_url.query)} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + params = super().request_params(stream_state, stream_slice, next_page_token) + return {**params, **next_page_token} if next_page_token else params + + +class Members(OrbitStreamPaginated): + # Docs: https://docs.orbit.love/reference/members-overview + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return f"{self.workspace}/members" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + params = super().request_params(stream_state, stream_slice, next_page_token) + params["sort"] = "created_at" + if self.start_date is not None: + params["start_date"] = self.start_date # The start_date parameter is filtering the last_activity_occurred_at field + return params + + +class Workspace(OrbitStream): + # Docs: https://docs.orbit.love/reference/get_workspaces-workspace-slug + # This stream is primarily used for connnection checking. + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return f"workspaces/{self.workspace}" + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + data = response.json() + yield data["data"] diff --git a/airbyte-integrations/connectors/source-orbit/unit_tests/__init__.py b/airbyte-integrations/connectors/source-orbit/unit_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-orbit/unit_tests/test_source.py b/airbyte-integrations/connectors/source-orbit/unit_tests/test_source.py new file mode 100644 index 0000000000000..1cdcf36126d4b --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/unit_tests/test_source.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import pytest +from source_orbit.source import SourceOrbit, Workspace + + +@pytest.mark.parametrize( + "read_records_side_effect, expected_return_value, expected_error_message", + [ + (iter(["foo", "bar"]), True, None), + ( + Exception("connection error"), + False, + "Please check that your API key and workspace name are entered correctly: Exception('connection error')", + ), + ], +) +def test_check_connection(mocker, read_records_side_effect, expected_return_value, expected_error_message): + source = SourceOrbit() + if expected_error_message: + read_records_mock = mocker.Mock(side_effect=read_records_side_effect) + else: + read_records_mock = mocker.Mock(return_value=read_records_side_effect) + mocker.patch.object(Workspace, "read_records", read_records_mock) + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (expected_return_value, expected_error_message) + + +def test_streams(mocker): + source = SourceOrbit() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-orbit/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-orbit/unit_tests/test_streams.py new file mode 100644 index 0000000000000..4c15591f6d03f --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/unit_tests/test_streams.py @@ -0,0 +1,98 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_orbit.streams import Members, OrbitStream, OrbitStreamPaginated + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(OrbitStream, "path", "v0/example_endpoint") + mocker.patch.object(OrbitStream, "primary_key", "test_primary_key") + mocker.patch.object(OrbitStream, "__abstractmethods__", set()) + mocker.patch.object(OrbitStreamPaginated, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = OrbitStream(workspace="workspace") + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = OrbitStream(workspace="workspace") + inputs = {"response": MagicMock()} + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class, mocker): + stream = OrbitStream(workspace="workspace") + inputs = {"response": mocker.Mock(json=mocker.Mock(return_value={"data": ["foo", "bar"]}))} + gen = stream.parse_response(**inputs) + assert next(gen) == "foo" + assert next(gen) == "bar" + + +def test_request_headers(patch_base_class): + stream = OrbitStream(workspace="workspace") + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = OrbitStream(workspace="workspace") + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = OrbitStream(workspace="workspace") + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = OrbitStream(workspace="workspace") + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time + + +class TestOrbitStreamPaginated: + @pytest.mark.parametrize( + "json_response, expected_token", [({"links": {"next": "http://foo.bar/api?a=b&c=d"}}, {"a": "b", "c": "d"}), ({}, None)] + ) + def test_next_page_token(self, patch_base_class, mocker, json_response, expected_token): + stream = OrbitStreamPaginated(workspace="workspace") + inputs = {"response": mocker.Mock(json=mocker.Mock(return_value=json_response))} + assert stream.next_page_token(**inputs) == expected_token + + +class TestMembers: + @pytest.mark.parametrize("start_date", [None, "2022-06-27"]) + def test_members_request_params(self, patch_base_class, start_date): + stream = Members(workspace="workspace", start_date=start_date) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + if start_date is not None: + expected_params = {"sort": "created_at", "start_date": start_date} + else: + expected_params = {"sort": "created_at"} + assert stream.request_params(**inputs) == expected_params diff --git a/docs/integrations/sources/orbit.md b/docs/integrations/sources/orbit.md new file mode 100644 index 0000000000000..ec28088a1991c --- /dev/null +++ b/docs/integrations/sources/orbit.md @@ -0,0 +1,48 @@ +# PostHog + +## Sync overview + +This source can sync data for the [PostHog API](https://docs.orbit.love/reference/about-the-orbit-api). It currently only supports Full Refresh syncs. + +### Output schema + +This Source is capable of syncing the following core Streams: + +* [Members](https://docs.orbit.love/reference/get_-workspace-slug-members) +* [Workspaces](https://docs.orbit.love/reference/get_workspaces-workspace-slug) + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | +| Namespaces | No | | +| Pagination | Yes | | + +### Performance considerations / Rate Limiting + +The Orbit API is rate limited at 120 requests per IP per minute as stated [here](https://docs.orbit.love/reference/rate-limiting). + +Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* Orbit API key - This can either be a workspace-tied key or a general personal key. + +### Setup guide + +The Orbit API Key should be available to you immediately as an Orbit user. + +1. Head to app.orbit.love and login to your account. +2. Go to the **Settings** tab on the right sidebar. +3. Navigate to **API Tokens**. +4. Click **New API Token** in the top right if one doesn't already exist. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | +| 0.1.0 | 2022-06-27 | [13390](https://github.com/airbytehq/airbyte/pull/13390) | Initial Release |