From acea2a54e5258059060f282431d5b4cd8548956d Mon Sep 17 00:00:00 2001 From: Himanshu Date: Thu, 17 Nov 2022 21:48:50 +0530 Subject: [PATCH] :tada: New source: Coda [python cdk] (#18675) * init coda * fix: Streams added * fix: Common review comments * fix: definition * fix: Remove sample files * fix: pagination * integration test configs * fix: remove validations * fix: unit test * fix: unit test removed * fix: unit tests added * linting * fix: pytest to 6.2.5 * remove columns * update connector * add coda to source def * remove coda from source def * readd missing file * rollback change to index.html * correct some files * rollback change in cart.com file * fix sample files * run format * auto-bump connector version Co-authored-by: marcosmarxm Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 7 + .../src/main/resources/seed/source_specs.yaml | 20 +++ .../connectors/source-coda/.dockerignore | 6 + .../connectors/source-coda/Dockerfile | 38 ++++ .../connectors/source-coda/README.md | 132 ++++++++++++++ .../source-coda/acceptance-test-config.yml | 37 ++++ .../source-coda/acceptance-test-docker.sh | 16 ++ .../connectors/source-coda/build.gradle | 9 + .../source-coda/integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 ++ .../integration_tests/configured_catalog.json | 67 +++++++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../connectors/source-coda/main.py | 13 ++ .../connectors/source-coda/requirements.txt | 2 + .../connectors/source-coda/setup.py | 29 +++ .../source-coda/source_coda/__init__.py | 8 + .../source_coda/schemas/categories.json | 9 + .../source_coda/schemas/controls.json | 39 ++++ .../source-coda/source_coda/schemas/docs.json | 153 ++++++++++++++++ .../source_coda/schemas/formulas.json | 39 ++++ .../source_coda/schemas/pages.json | 164 +++++++++++++++++ .../source_coda/schemas/permissions.json | 23 +++ .../source_coda/schemas/tables.json | 44 +++++ .../source-coda/source_coda/source.py | 170 ++++++++++++++++++ .../source-coda/source_coda/spec.yaml | 15 ++ .../source-coda/unit_tests/__init__.py | 3 + .../source-coda/unit_tests/test_source.py | 52 ++++++ .../source-coda/unit_tests/test_streams.py | 91 ++++++++++ docs/integrations/sources/coda.md | 62 +++++++ 32 files changed, 1283 insertions(+) create mode 100644 airbyte-integrations/connectors/source-coda/.dockerignore create mode 100644 airbyte-integrations/connectors/source-coda/Dockerfile create mode 100644 airbyte-integrations/connectors/source-coda/README.md create mode 100644 airbyte-integrations/connectors/source-coda/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-coda/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-coda/build.gradle create mode 100644 airbyte-integrations/connectors/source-coda/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-coda/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-coda/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-coda/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-coda/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-coda/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-coda/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-coda/main.py create mode 100644 airbyte-integrations/connectors/source-coda/requirements.txt create mode 100644 airbyte-integrations/connectors/source-coda/setup.py create mode 100755 airbyte-integrations/connectors/source-coda/source_coda/__init__.py create mode 100644 airbyte-integrations/connectors/source-coda/source_coda/schemas/categories.json create mode 100644 airbyte-integrations/connectors/source-coda/source_coda/schemas/controls.json create mode 100644 airbyte-integrations/connectors/source-coda/source_coda/schemas/docs.json create mode 100644 airbyte-integrations/connectors/source-coda/source_coda/schemas/formulas.json create mode 100644 airbyte-integrations/connectors/source-coda/source_coda/schemas/pages.json create mode 100644 airbyte-integrations/connectors/source-coda/source_coda/schemas/permissions.json create mode 100644 airbyte-integrations/connectors/source-coda/source_coda/schemas/tables.json create mode 100755 airbyte-integrations/connectors/source-coda/source_coda/source.py create mode 100755 airbyte-integrations/connectors/source-coda/source_coda/spec.yaml create mode 100644 airbyte-integrations/connectors/source-coda/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-coda/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-coda/unit_tests/test_streams.py create mode 100755 docs/integrations/sources/coda.md diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e22188313865e..d59e86658eae4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -254,6 +254,13 @@ icon: cockroachdb.svg sourceType: database releaseStage: alpha +- name: Coda + sourceDefinitionId: 27f910fd-f832-4b2e-bcfd-6ab342e434d8 + dockerRepository: airbyte/source-coda + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/coda + sourceType: api + releaseStage: alpha - name: Coin API sourceDefinitionId: 919984ef-53a2-479b-8ffe-9c1ddb9fc3f3 dockerRepository: airbyte/source-coin-api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 2381a6f4d4288..1e8fc4842e0ea 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2415,6 +2415,26 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-coda:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/coda" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Required attributes for hitting apis" + type: "object" + required: + - "auth_token" + additionalProperties: true + properties: + auth_token: + title: "Authentication token" + type: "string" + description: "Bearer token" + airbyte_secret: true + order: 0 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-coin-api:0.1.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/coin-api" diff --git a/airbyte-integrations/connectors/source-coda/.dockerignore b/airbyte-integrations/connectors/source-coda/.dockerignore new file mode 100644 index 0000000000000..b79a53f6d7af9 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_coda +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-coda/Dockerfile b/airbyte-integrations/connectors/source-coda/Dockerfile new file mode 100644 index 0000000000000..f8d116a3a9246 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_coda ./source_coda + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-coda diff --git a/airbyte-integrations/connectors/source-coda/README.md b/airbyte-integrations/connectors/source-coda/README.md new file mode 100644 index 0000000000000..85b13c68d5616 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/README.md @@ -0,0 +1,132 @@ +# Coda Source + +This is the repository for the Coda source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/coda). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-coda:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/coda) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_coda/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source coda test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-coda:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-coda:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-coda:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coda:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coda:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-coda:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-coda:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-coda:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-coda/acceptance-test-config.yml b/airbyte-integrations/connectors/source-coda/acceptance-test-config.yml new file mode 100644 index 0000000000000..efb1873158325 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/acceptance-test-config.yml @@ -0,0 +1,37 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-coda:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_coda/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: formulas + bypass_reason: "no records" + - name: permissions + bypass_reason: "no records" +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-coda/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-coda/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-coda/build.gradle b/airbyte-integrations/connectors/source-coda/build.gradle new file mode 100644 index 0000000000000..d685c0c4bc743 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_coda' +} diff --git a/airbyte-integrations/connectors/source-coda/integration_tests/__init__.py b/airbyte-integrations/connectors/source-coda/integration_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-coda/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-coda/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..52b0f2c2118f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-coda/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-coda/integration_tests/acceptance.py new file mode 100644 index 0000000000000..1302b2f57e10e --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-coda/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-coda/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..b6bef3c3fd7f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/integration_tests/configured_catalog.json @@ -0,0 +1,67 @@ +{ + "streams": [ + { + "stream": { + "name": "docs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "permissions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "categories", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "pages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "tables", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "formulas", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "controls", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-coda/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-coda/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..f8cbdf072c870 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "auth_token": "invalid authentication token" +} diff --git a/airbyte-integrations/connectors/source-coda/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-coda/integration_tests/sample_config.json new file mode 100644 index 0000000000000..cbbdb13d6e975 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "auth_token": "1111102a999ce-a634-42a4-a0b9-90324c436b6a" +} diff --git a/airbyte-integrations/connectors/source-coda/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-coda/integration_tests/sample_state.json new file mode 100644 index 0000000000000..3587e579822d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-coda/main.py b/airbyte-integrations/connectors/source-coda/main.py new file mode 100644 index 0000000000000..0a5c909b0d31a --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_coda import SourceCoda + +if __name__ == "__main__": + source = SourceCoda() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-coda/requirements.txt b/airbyte-integrations/connectors/source-coda/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-coda/setup.py b/airbyte-integrations/connectors/source-coda/setup.py new file mode 100644 index 0000000000000..574189c1bced7 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.2.5", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_coda", + description="Source implementation for Coda.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-coda/source_coda/__init__.py b/airbyte-integrations/connectors/source-coda/source_coda/__init__.py new file mode 100755 index 0000000000000..2b7dca0819d94 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceCoda + +__all__ = ["SourceCoda"] diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/categories.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/categories.json new file mode 100644 index 0000000000000..a90ed1fede997 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/categories.json @@ -0,0 +1,9 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/controls.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/controls.json new file mode 100644 index 0000000000000..14fc1e737c60b --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/controls.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + + "href": { + "type": "string" + }, + "name": { + "type": "string" + }, + "parent": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "id": { + "type": "string" + }, + "href": { + "type": "string" + }, + "browserLink": { + "type": "string" + }, + "name": { + "type": "string" + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/docs.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/docs.json new file mode 100644 index 0000000000000..3a28e58b0c336 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/docs.json @@ -0,0 +1,153 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "href": { + "type": "string" + }, + "browserLink": { + "type": "string" + }, + "icon": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "browserLink": { + "type": "string" + } + } + }, + "name": { + "type": "string" + }, + "owner": { + "type": "string" + }, + "owner_name": { + "type": "string" + }, + "docSize": { + "type": "object", + "properties": { + "totalRowCount": { + "type": "integer" + }, + "tableViewCount": { + "type": "integer" + }, + "pageCount": { + "type": "integer" + }, + "overApiSizeLimit": { + "type": "boolean" + } + } + }, + "sourceDoc": { + "type": "object", + "properties": { + "totalRowCount": { + "type": "string" + }, + "type": { + "type": "string" + }, + "href": { + "type": "string" + }, + "browserLink": { + "type": "string" + } + } + }, + "createdAt": { + "type": "string" + }, + "updatedAt": { + "type": "string" + }, + "published": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "browserLink": { + "type": "string" + }, + "imageLink": { + "type": "string" + }, + "discoverable": { + "type": "boolean" + }, + "earnCredit": { + "type": "boolean" + }, + "mode": { + "type": "string" + }, + "categories": { + "type": "string", + "items": { + "type": "string" + } + } + } + }, + "folder": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "browserLink": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "workspace": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "organizationId": { + "type": "string" + }, + "browserLink": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "workspaceId": { + "type": "string" + }, + "folderId": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/formulas.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/formulas.json new file mode 100644 index 0000000000000..14fc1e737c60b --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/formulas.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + + "href": { + "type": "string" + }, + "name": { + "type": "string" + }, + "parent": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "id": { + "type": "string" + }, + "href": { + "type": "string" + }, + "browserLink": { + "type": "string" + }, + "name": { + "type": "string" + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/pages.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/pages.json new file mode 100644 index 0000000000000..79ca344a2b135 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/pages.json @@ -0,0 +1,164 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "href": { + "type": "string" + }, + "browserLink": { + "type": "string" + }, + "name": { + "type": "string" + }, + "subtitle": { + "type": "string" + }, + "icon": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "browserLink": { + "type": "string" + } + } + }, + "image": { + "type": "object", + "properties": { + "width": { + "type": "number" + }, + "height": { + "type": "number" + }, + "type": { + "type": "string" + }, + "browserLink": { + "type": "string" + } + } + }, + "parent": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "href": { + "type": "string" + }, + "browserLink": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "children": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "href": { + "type": "string" + }, + "browserLink": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, + "authors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "@context": { + "type": "string" + }, + "@type": { + "type": "string" + }, + "additionalType": { + "type": "string" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + }, + "createdAt": { + "type": "string" + }, + "updatedBy": { + "type": "object", + "properties": { + "@context": { + "type": "string" + }, + "@type": { + "type": "string" + }, + "additionalType": { + "type": "string" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + }, + "createdBy": { + "type": "object", + "properties": { + "@context": { + "type": "string" + }, + "@type": { + "type": "string" + }, + "additionalType": { + "type": "string" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/permissions.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/permissions.json new file mode 100644 index 0000000000000..e440e36472a4d --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/permissions.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "access": { + "type": "string" + }, + "principal": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "email": { + "type": "string" + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-coda/source_coda/schemas/tables.json b/airbyte-integrations/connectors/source-coda/source_coda/schemas/tables.json new file mode 100644 index 0000000000000..94b46a4170dcf --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/schemas/tables.json @@ -0,0 +1,44 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "tableType": { + "type": "string" + }, + "href": { + "type": "string" + }, + "browserLink": { + "type": "string" + }, + "name": { + "type": "string" + }, + "parent": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "href": { + "type": "string" + }, + "browserLink": { + "type": "string" + }, + "name": { + "type": "string" + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-coda/source_coda/source.py b/airbyte-integrations/connectors/source-coda/source_coda/source.py new file mode 100755 index 0000000000000..8bd60251c9cfb --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/source.py @@ -0,0 +1,170 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +BASE_URL = "https://coda.io/apis/v1/" + + +# Basic full refresh stream +class CodaStream(HttpStream, ABC): + + url_base = BASE_URL + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.limit = 25 + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return response.json().get("nextPageToken", None) + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + if next_page_token: + return {"pageToken": next_page_token, "limit": self.limit} + else: + return {"limit": self.limit} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json()["items"] + + +class Docs(CodaStream): + + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "docs" + + +class CodaStreamDoc(CodaStream): + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + """ + self.authenticator (which should be used as the + authenticator for Users) is object of NoAuth() + + so self._session.auth is used instead + """ + docs_stream = Docs(**{"authenticator": self._authenticator}) + for doc in docs_stream.read_records(sync_mode=SyncMode.full_refresh): + yield {"doc_id": doc["id"]} + + +class Permissions(CodaStreamDoc): + + primary_key = "id" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + doc_id = stream_slice["doc_id"] + return f"docs/{doc_id}/acl/permissions" + + +class Categories(CodaStreamDoc): + + primary_key = "name" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "categories" + + +class Pages(CodaStreamDoc): + + primary_key = "id" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + doc_id = stream_slice["doc_id"] + return f"docs/{doc_id}/pages" + + +class Tables(CodaStreamDoc): + + primary_key = "id" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + doc_id = stream_slice["doc_id"] + return f"docs/{doc_id}/tables" + + +class Formulas(CodaStreamDoc): + + primary_key = "id" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + doc_id = stream_slice["doc_id"] + return f"docs/{doc_id}/formulas" + + +class Controls(CodaStreamDoc): + + primary_key = "id" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + doc_id = stream_slice["doc_id"] + return f"docs/{doc_id}/controls" + + +# Source +class SourceCoda(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + try: + token = config.get("auth_token") + headers = {"Authorization": f"Bearer {token}"} + r = requests.get(f"{BASE_URL}whoami", headers=headers) + if r.status_code == 200: + return True, None + except Exception as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + stream_args = { + "authenticator": TokenAuthenticator(token=config.get("auth_token")), + } + + return [ + Docs(**stream_args), + Permissions(**stream_args), + Categories(**stream_args), + Pages(**stream_args), + Tables(**stream_args), + Formulas(**stream_args), + Controls(**stream_args), + ] diff --git a/airbyte-integrations/connectors/source-coda/source_coda/spec.yaml b/airbyte-integrations/connectors/source-coda/source_coda/spec.yaml new file mode 100755 index 0000000000000..138ab5bfc046e --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/source_coda/spec.yaml @@ -0,0 +1,15 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/coda +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Required attributes for hitting apis + type: object + required: + - auth_token + additionalProperties: true + properties: + auth_token: + title: Authentication token + type: string + description: Bearer token + airbyte_secret: true + order: 0 diff --git a/airbyte-integrations/connectors/source-coda/unit_tests/__init__.py b/airbyte-integrations/connectors/source-coda/unit_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-coda/unit_tests/test_source.py b/airbyte-integrations/connectors/source-coda/unit_tests/test_source.py new file mode 100644 index 0000000000000..1e33b4e639018 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/unit_tests/test_source.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock, patch + +from source_coda.source import SourceCoda + + +class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + def raise_for_status(self): + if self.status_code != 200: + raise Exception("Bad things happened") + + +def mocked_requests_get(fail=False): + def wrapper(*args, **kwargs): + if fail: + return MockResponse(None, 404) + + return MockResponse( + {"name": "John", "loginId": "john@example.com", "type": "user", "href": "https://coda.io/apis/v1/whoami", "tokenName": "as", "scoped": False, "pictureLink": "https://images-coda.io", "workspace":{ + "id": "test-id", + "type": "workspace", + "browserLink": "https://coda.io/link", + "name": "title" + }}, 200 + ) + + return wrapper + + +@patch("requests.get", side_effect=mocked_requests_get()) +def test_check_connection(mocker): + source = SourceCoda() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceCoda() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 7 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-coda/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-coda/unit_tests/test_streams.py new file mode 100644 index 0000000000000..7f1a96c57e050 --- /dev/null +++ b/airbyte-integrations/connectors/source-coda/unit_tests/test_streams.py @@ -0,0 +1,91 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from source_coda.source import CodaStream + +logger = logging.getLogger() +logger.level = logging.DEBUG + + +authenticator = TokenAuthenticator(token="test_token"), + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(CodaStream, "path", "v0/example_endpoint") + mocker.patch.object(CodaStream, "primary_key", "test_primary_key") + mocker.patch.object(CodaStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = CodaStream() + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {"limit": 25} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = CodaStream(authenticator=authenticator) + response = MagicMock() + response.json.return_value = { + "id": "1244fds", + "name": "Test doc" + } + inputs = {"response": response} + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = CodaStream(authenticator=authenticator) + + response = MagicMock() + response.json = MagicMock(return_value={'items': [{"id": 101}]}) + + inputs = {"response": response} + expected_parsed_object = response.json()['items'][0] + assert next(iter(stream.parse_response(**inputs))) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = CodaStream(authenticator=authenticator) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = CodaStream(authenticator=authenticator) + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = CodaStream(authenticator=authenticator) + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = CodaStream(authenticator=authenticator) + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/sources/coda.md b/docs/integrations/sources/coda.md new file mode 100755 index 0000000000000..301f22536f3b5 --- /dev/null +++ b/docs/integrations/sources/coda.md @@ -0,0 +1,62 @@ +# Coda + +This page contains the setup guide and reference information for the Coda source connector. + +## Prerequisites + +You can find or create authentication tokens within [Coda](https://coda.io/account#apiSettings). + +## Setup guide +## Step 1: Set up the Coda connector in Airbyte + +### For Airbyte Cloud: + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. +3. On the Set up the source page, enter the name for the Coda connector and select **Coda** from the Source type dropdown. +4. Enter your `auth_token` - Coda Authentication Token with the necessary permissions \(described below\). +5. Enter your `doc_id` - Document id for a specific document created on Coda. You can check it under [Advanced Settings](https://coda.io/account) + by exporting data and copying the id in doc_manifest.json from the downloaded zip. +6. Click **Set up source**. + +### For Airbyte OSS: + +1. Navigate to the Airbyte Open Source dashboard. +2. Set the name for your source. +3. Enter your `auth_token` - Coda Authentication Token with the necessary permissions \(described below\). +4. Enter your `doc_id` - Document id for a specific document created on Coda. You can check it under [Advanced Settings](https://coda.io/account) + by exporting data and copying the id in doc_manifest.json from the downloaded zip. +5. Click **Set up source**. + +## Supported sync modes + +The Coda source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +| Feature | Supported? | +| :---------------- |:-----------| +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| SSL connection | No | +| Namespaces | No | + +## Supported Streams + +* [Docs](https://coda.io/developers/apis/v1#tag/Docs/operation/listDocs) +* [Permissions](https://coda.io/developers/apis/v1#tag/Permissions/operation/getPermissions) +* [Categories](https://coda.io/developers/apis/v1#tag/Publishing/operation/listCategories) +* [Pages](https://coda.io/developers/apis/v1#tag/Pages/operation/listPages) +* [Tables](https://coda.io/developers/apis/v1#tag/Tables/operation/listTables) +* [Formulas](https://coda.io/developers/apis/v1#tag/Formulas/operation/listFormulas) +* [Controls](https://coda.io/developers/apis/v1#tag/Controls/operation/listControls) + + + +## Data type map + +| Integration Type | Airbyte Type | +|:-----------------| :----------- | +| `string` | `string` | +| `integer` | `number` | +| `array` | `array` | +| `object` | `object` | +