diff --git a/airbyte-config/init/src/main/resources/icons/chartmogul.svg b/airbyte-config/init/src/main/resources/icons/chartmogul.svg new file mode 100644 index 0000000000000..4957ba4e9b413 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/chartmogul.svg @@ -0,0 +1,13 @@ + + + + + + diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 6ce13683cf803..2d86e3f7b999a 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -116,6 +116,13 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/chargebee icon: chargebee.svg sourceType: api +- name: Chartmogul + sourceDefinitionId: b6604cbd-1b12-4c08-8767-e140d0fb0877 + dockerRepository: airbyte/source-chartmogul + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/chartmogul + icon: chartmogul.svg + sourceType: api - name: ClickHouse sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c dockerRepository: airbyte/source-clickhouse diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index ecf1dee8fdd65..005c5af601381 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1000,6 +1000,33 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-chartmogul:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/chartmogul" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Chartmogul Spec" + type: "object" + required: + - "api_key" + additionalProperties: false + properties: + api_key: + type: "string" + description: "Chartmogul API key" + airbyte_secret: true + order: 0 + start_date: + type: "string" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + description: "UTC date and time in the format 2017-01-25T00:00:00Z. When\ + \ feasible, any data before this date will not be replicated." + examples: + - "2017-01-25T00:00:00Z" + order: 1 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-clickhouse:0.1.7" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 001ae80fd3248..c1adc5af18e50 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -20,6 +20,7 @@ | BigQuery | [![source-bigquery](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-bigquery%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-bigquery/) | | Bing Ads | [![source-bing-ads](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-bing-ads%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-bing-ads) | | Chargebee | [![source-chargebee](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-chargebee%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-chargebee/) | +| Chartmogul | [![source-chartmogul](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-chartmogul%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-chartmogul/) | | Cart.com | [![source-cart](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-cart%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-cart/) | | Close.com | [![source-close-com](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-close-com%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-close-com/) | | Dixa | [![source-dixa](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-dixa%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-dixa) | diff --git a/airbyte-integrations/connectors/source-chartmogul/Dockerfile b/airbyte-integrations/connectors/source-chartmogul/Dockerfile new file mode 100644 index 0000000000000..35cef5c730227 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_chartmogul ./source_chartmogul + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-chartmogul diff --git a/airbyte-integrations/connectors/source-chartmogul/README.md b/airbyte-integrations/connectors/source-chartmogul/README.md new file mode 100644 index 0000000000000..2ba338da1728d --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/README.md @@ -0,0 +1,132 @@ +# Chartmogul Source + +This is the repository for the Chartmogul source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/chartmogul). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python3 -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-chartmogul:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/chartmogul) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_chartmogul/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source chartmogul test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-chartmogul:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-chartmogul:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-chartmogul:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-chartmogul:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-chartmogul:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-chartmogul:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-chartmogul:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-chartmogul:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-chartmogul/acceptance-test-config.yml b/airbyte-integrations/connectors/source-chartmogul/acceptance-test-config.yml new file mode 100644 index 0000000000000..ba3029611366a --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-chartmogul:dev +tests: + spec: + - spec_path: "source_chartmogul/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: ["activities"] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-chartmogul/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-chartmogul/acceptance-test-docker.sh new file mode 100755 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-chartmogul/bootstrap.md b/airbyte-integrations/connectors/source-chartmogul/bootstrap.md new file mode 100644 index 0000000000000..d6fee29243dad --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/bootstrap.md @@ -0,0 +1,21 @@ +# Chartmogul +Chartmogul is an online subscription analytics platform. It retrieves data from payment processors (e.g. Stripe) and makes sense out of it. + +## Streams + +Connector currently implements following full refresh streams: +* [Customers](https://dev.chartmogul.com/reference/list-customers) +* [Activities](https://dev.chartmogul.com/reference/list-activities) + +`start_date` config is used for retrieving `Activies`. `Customers` stream does not use this config. Even if it was possible to filter by `start_date`, it would cause issues when modeling data. That is because activies after `start_date` can be triggered by customers who were created way before that. + +### Incremental streams +Incremental streams were not implemented due to following reasons: +* `Customers` API endpoint does not provide filtering by creation/update date. +* `Activities` API does provide pagination based on last entries UUID, however it is not stable, since it is possible to for activity to disappear retrospectively. + +### Next steps +It is theoretically possible to make `Activities` stream incremental. One would need to keep track of both UUID and created_at and read stream until `datetime.now()`. Dynamic end date would be necessary since activities can also have a future date. Since data can be changed retrospectively, a `lookback window` would also be necessary to catch all the changes. + +### Rate limits +The API rate limit is at 40 requests/second. Read [Rate Limits](https://dev.chartmogul.com/docs/rate-limits) for more informations. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-chartmogul/build.gradle b/airbyte-integrations/connectors/source-chartmogul/build.gradle new file mode 100644 index 0000000000000..0f9c12cdb8203 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_chartmogul' +} diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/__init__.py b/airbyte-integrations/connectors/source-chartmogul/integration_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-chartmogul/integration_tests/acceptance.py new file mode 100644 index 0000000000000..0347f2a0b143d --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-chartmogul/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..3b788b27041d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "activities", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..dc521ade7acf9 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "" +} diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-chartmogul/integration_tests/sample_config.json new file mode 100644 index 0000000000000..c09fca36f4670 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "", + "start_date": "2022-01-05T12:09:00Z" +} diff --git a/airbyte-integrations/connectors/source-chartmogul/main.py b/airbyte-integrations/connectors/source-chartmogul/main.py new file mode 100644 index 0000000000000..89e0ffbb29778 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_chartmogul import SourceChartmogul + +if __name__ == "__main__": + source = SourceChartmogul() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-chartmogul/requirements.txt b/airbyte-integrations/connectors/source-chartmogul/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-chartmogul/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-chartmogul/sample_files/configured_catalog.json new file mode 100644 index 0000000000000..3b788b27041d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/sample_files/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "activities", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-chartmogul/setup.py b/airbyte-integrations/connectors/source-chartmogul/setup.py new file mode 100644 index 0000000000000..446c4426fb9bb --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", + "requests-mock", +] + +setup( + name="source_chartmogul", + description="Source implementation for Chartmogul.", + author="Titas Skrebe", + author_email="titas@omnisend.com", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/__init__.py b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/__init__.py new file mode 100644 index 0000000000000..9d8418296529d --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceChartmogul + +__all__ = ["SourceChartmogul"] diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/activities.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/activities.json new file mode 100644 index 0000000000000..6cf82203be202 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/activities.json @@ -0,0 +1,48 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "activity-mrr-movement": { + "type": ["null", "integer"] + }, + "activity-mrr": { + "type": ["null", "integer"] + }, + "activity-arr": { + "type": ["null", "integer"] + }, + "date": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "subscription-external-id": { + "type": ["null", "string"] + }, + "plan-external-id": { + "type": ["null", "string"] + }, + "customer-name": { + "type": ["null", "string"] + }, + "customer-uuid": { + "type": ["null", "string"] + }, + "customer-external-id": { + "type": ["null", "string"] + }, + "billing-connector-uuid": { + "type": ["null", "string"] + }, + "uuid": { + "type": ["string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customers.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customers.json new file mode 100644 index 0000000000000..eb499b01a0d83 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customers.json @@ -0,0 +1,128 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["integer"] + }, + "uuid": { + "type": ["string"] + }, + "external_id": { + "type": ["null", "string"] + }, + "external_ids": { + "type": ["null", "array"] + }, + "data_source_uuid": { + "type": ["null", "string"] + }, + "data_source_uuids": { + "type": ["null", "array"] + }, + "name": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "lead_created_at": { + "type": ["null", "string"] + }, + "free_trial_started_at": { + "type": ["null", "string"] + }, + "customer_since": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "attributes": { + "type": ["null", "object"], + "properties": { + "tags": { + "type": ["null", "array"] + }, + "stripe": { + "type": ["null", "object"], + "properties": { + "brandID": { + "type": ["null", "string"] + }, + "brandName": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"] + }, + "platformName": { + "type": ["null", "string"] + } + } + }, + "clearbit": { + "type": ["null", "object"], + "properties": {} + }, + "custom": { + "type": ["null", "object"], + "properties": {} + } + } + }, + "address": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "address_zip": { + "type": ["null", "string"] + } + } + }, + "mrr": { + "type": ["null", "integer"] + }, + "arr": { + "type": ["null", "integer"] + }, + "billing-system-url": { + "type": ["null", "string"] + }, + "chartmogul-url": { + "type": ["null", "string"] + }, + "billing-system-type": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "currency-sign": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py new file mode 100644 index 0000000000000..6a3a8fb675fe2 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from base64 import b64encode +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + + +# Basic full refresh stream +class ChartmogulStream(HttpStream, ABC): + url_base = "https://api.chartmogul.com" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json().get("entries", []) + + +class Customers(ChartmogulStream): + primary_key = "id" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + json_response = response.json() + if json_response.get("has_more", False): + return {"page": json_response.get("current_page") + 1} + + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {"page": 1 if not next_page_token else next_page_token["page"]} + + def path(self, **kwargs) -> str: + return "v1/customers" + + +class Activities(ChartmogulStream): + primary_key = "uuid" + + def __init__(self, start_date: str, **kwargs): + super().__init__(**kwargs) + self.start_date = start_date + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + json_response = response.json() + if not json_response.get("has_more", False): + return None + + return {"start-after": json_response["entries"][-1][self.primary_key]} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = {} + + if next_page_token: + params.update(next_page_token) + elif self.start_date: + params["start-date"] = self.start_date + + return params + + def path(self, **kwargs) -> str: + return "v1/activities" + + +class HttpBasicAuthenticator(TokenAuthenticator): + def __init__(self, token: str, auth_method: str = "Basic", **kwargs): + auth_string = f"{token}:".encode("utf8") + b64_encoded = b64encode(auth_string).decode("utf8") + super().__init__(token=b64_encoded, auth_method=auth_method, **kwargs) + + +# Source +class SourceChartmogul(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + auth = HttpBasicAuthenticator(config["api_key"], auth_method="Basic").get_auth_header() + url = f"{ChartmogulStream.url_base}/v1/ping" + try: + resp = requests.get(url, headers=auth) + resp.raise_for_status() + return True, None + except Exception as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + auth = HttpBasicAuthenticator(config["api_key"], auth_method="Basic") + return [Customers(authenticator=auth), Activities(authenticator=auth, start_date=config.get("start_date"))] diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json new file mode 100644 index 0000000000000..d6f16407e70ce --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json @@ -0,0 +1,25 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/chartmogul", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Chartmogul Spec", + "type": "object", + "required": ["api_key"], + "additionalProperties": false, + "properties": { + "api_key": { + "type": "string", + "description": "Chartmogul API key", + "airbyte_secret": true, + "order": 0 + }, + "start_date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. When feasible, any data before this date will not be replicated.", + "examples": ["2017-01-25T00:00:00Z"], + "order": 1 + } + } + } +} diff --git a/airbyte-integrations/connectors/source-chartmogul/unit_tests/__init__.py b/airbyte-integrations/connectors/source-chartmogul/unit_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-chartmogul/unit_tests/test_source.py b/airbyte-integrations/connectors/source-chartmogul/unit_tests/test_source.py new file mode 100644 index 0000000000000..32b7774924343 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/unit_tests/test_source.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from requests.exceptions import HTTPError +from source_chartmogul.source import SourceChartmogul + + +def test_check_connection(mocker, requests_mock): + source = SourceChartmogul() + logger_mock, config_mock = MagicMock(), MagicMock() + + # success + requests_mock.get("https://api.chartmogul.com/v1/ping", json={"data": "pong!"}) + assert source.check_connection(logger_mock, config_mock) == (True, None) + + # failure + requests_mock.get("https://api.chartmogul.com/v1/ping", status_code=500) + ok, err = source.check_connection(logger_mock, config_mock) + assert (ok, type(err)) == (False, HTTPError) + + +def test_streams(mocker): + source = SourceChartmogul() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-chartmogul/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-chartmogul/unit_tests/test_streams.py new file mode 100644 index 0000000000000..fc11b843aba87 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/unit_tests/test_streams.py @@ -0,0 +1,84 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import pytest +from source_chartmogul.source import Activities, Customers + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(Customers, "__abstractmethods__", set()) + mocker.patch.object(Activities, "__abstractmethods__", set()) + + +class TestCustomers: + def test_request_params(self): + stream = Customers() + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {"page": 1} + assert stream.request_params(**inputs) == expected_params + + next_page_token = {"page": 3} + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": next_page_token} + expected_params = {"page": 3} + assert stream.request_params(**inputs) == expected_params + + def test_next_page_token(self, mocker): + stream = Customers() + response = mocker.MagicMock() + + # no more results + response.json.return_value = {"has_more": False} + inputs = {"response": response} + assert stream.next_page_token(**inputs) is None + + # there is more results + response.json.return_value = {"has_more": True, "current_page": 42} + inputs = {"response": response} + assert stream.next_page_token(**inputs) == {"page": 43} + + def test_parse_response(self, mocker): + stream = Customers() + response = mocker.MagicMock() + response.json.return_value = {"entries": [{"one": 1}, {"two": 2}]} + inputs = {"response": response} + expected_parsed_object = {"one": 1} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +# Activites stream tests + + +class TestActivities: + def test_request_params(self): + # no start_date set + stream = Activities(start_date=None) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + assert stream.request_params(**inputs) == {} + + # start_date is set + stream.start_date = "2010-01-01" + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + assert stream.request_params(**inputs) == {"start-date": stream.start_date} + + # start-after is available + next_page_token = {"start-after": "a-b-c"} + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": next_page_token} + expected_params = next_page_token + assert stream.request_params(**inputs) == expected_params + + def test_next_page_token(self, mocker): + stream = Activities(start_date=None) + response = mocker.MagicMock() + + # no more results + response.json.return_value = {"has_more": False} + inputs = {"response": response} + assert stream.next_page_token(**inputs) is None + + # there is more results + response.json.return_value = {"has_more": True, "entries": [{"uuid": "unique-uuid"}]} + inputs = {"response": response} + assert stream.next_page_token(**inputs) == {"start-after": "unique-uuid"} diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 56c36d6dec8e2..f37b28be05589 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -51,6 +51,7 @@ * [Braintree](integrations/sources/braintree.md) * [Cart](integrations/sources/cart.md) * [Chargebee](integrations/sources/chargebee.md) + * [Chartmogul](integrations/sources/chartmogul.md) * [ClickHouse](integrations/sources/clickhouse.md) * [Close.com](integrations/sources/close-com.md) * [CockroachDB](integrations/sources/cockroachdb.md) diff --git a/docs/integrations/README.md b/docs/integrations/README.md index f91687ce224ae..91b6163db9d6b 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -30,6 +30,7 @@ Airbyte uses a grading system for connectors to help users understand what to ex | [Bing Ads](sources/bing-ads.md) | Beta | | [Cart.com](sources/cart.md) | Beta | | [Chargebee](sources/chargebee.md) | Alpha | +| [Chartmogul](sources/chartmogul.md) | Alpha | | [ClickHouse](sources/clickhouse.md) | Beta | | [Close.com](sources/close-com.md) | Beta | | [CockroachDB](sources/cockroachdb.md) | Beta | diff --git a/docs/integrations/sources/chartmogul.md b/docs/integrations/sources/chartmogul.md new file mode 100644 index 0000000000000..e8d19c2ddda27 --- /dev/null +++ b/docs/integrations/sources/chartmogul.md @@ -0,0 +1,47 @@ +# Chartmogul + +## Sync overview + +The Chartmogul source supports Full Refresh syncs only. + +This source syncs data for the [Chartmogul API](https://dev.chartmogul.com/reference/). + +### Notes + +If `start_date` is set, it will only apply to `Activities` stream. `Customers`' endpoint does not provide a way to filter by creation or update dates. + +### Output schema + +This Source is capable of syncing the following streams: + +* [Customers](https://dev.chartmogul.com/reference/list-customers) +* [Activities](https://dev.chartmogul.com/reference/list-activities) + +### Features + +| Feature | Supported?\(Yes/No\) +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | No | +| Namespaces | No | + +### Performance considerations + +The Chartmogul connector should not run into Chartmogul API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* Chartmogul Account +* Chartmogul API Key + +### Setup guide + +Please read [How to find your API key](https://dev.chartmogul.com/docs/authentication). + +## Changelog + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | +| 0.1.0 | 2022-01-10 | [9381](https://github.com/airbytehq/airbyte/pull/9381) | New Source: Chartmogul |