From 1403c1b84db51b4a44e93595aa0e319b9509bbe1 Mon Sep 17 00:00:00 2001 From: Scott Lee Chua <51027688+scottleechua@users.noreply.github.com> Date: Wed, 9 Nov 2022 08:29:05 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20New=20Source:=20EmailOctopus=20[?= =?UTF-8?q?low-code=20cdk]=20(#18647)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial commit - emailoctopus * Add API key parameter * Update schemas * Annotate schemas * Update integration tests * Acceptance test bug fixes * Change full refresh append to overwrite; delete abnormal state * Add documentation * Update documentation * Update documentation * Update DocumentationUrl, add titles to spec properties, remove schemas/TODO, remove EOF newlines * Add BOOTSTRAP.md * Remove todos * run format * add emailloctopus to source def * run format * auto-bump connector version Co-authored-by: marcosmarxm Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 7 ++ .../src/main/resources/seed/source_specs.yaml | 20 +++++ .../source-emailoctopus/.dockerignore | 6 ++ .../source-emailoctopus/BOOTSTRAP.md | 8 ++ .../connectors/source-emailoctopus/Dockerfile | 38 +++++++++ .../connectors/source-emailoctopus/README.md | 79 +++++++++++++++++++ .../source-emailoctopus/__init__.py | 3 + .../acceptance-test-config.yml | 27 +++++++ .../acceptance-test-docker.sh | 15 ++++ .../source-emailoctopus/build.gradle | 9 +++ .../integration_tests/__init__.py | 3 + .../integration_tests/acceptance.py | 14 ++++ .../integration_tests/configured_catalog.json | 34 ++++++++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../connectors/source-emailoctopus/main.py | 13 +++ .../source-emailoctopus/requirements.txt | 2 + .../connectors/source-emailoctopus/setup.py | 29 +++++++ .../source_emailoctopus/__init__.py | 8 ++ .../source_emailoctopus/emailoctopus.yaml | 76 ++++++++++++++++++ .../schemas/campaigns.json | 62 +++++++++++++++ .../source_emailoctopus/schemas/lists.json | 76 ++++++++++++++++++ .../source_emailoctopus/source.py | 18 +++++ .../source_emailoctopus/spec.yaml | 17 ++++ docs/integrations/sources/emailoctopus.md | 27 +++++++ 25 files changed, 597 insertions(+) create mode 100644 airbyte-integrations/connectors/source-emailoctopus/.dockerignore create mode 100644 airbyte-integrations/connectors/source-emailoctopus/BOOTSTRAP.md create mode 100644 airbyte-integrations/connectors/source-emailoctopus/Dockerfile create mode 100644 airbyte-integrations/connectors/source-emailoctopus/README.md create mode 100644 airbyte-integrations/connectors/source-emailoctopus/__init__.py create mode 100644 airbyte-integrations/connectors/source-emailoctopus/acceptance-test-config.yml create mode 100755 airbyte-integrations/connectors/source-emailoctopus/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-emailoctopus/build.gradle create mode 100644 airbyte-integrations/connectors/source-emailoctopus/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-emailoctopus/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-emailoctopus/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-emailoctopus/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-emailoctopus/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-emailoctopus/main.py create mode 100644 airbyte-integrations/connectors/source-emailoctopus/requirements.txt create mode 100644 airbyte-integrations/connectors/source-emailoctopus/setup.py create mode 100644 airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/__init__.py create mode 100644 airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/emailoctopus.yaml create mode 100644 airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/schemas/campaigns.json create mode 100644 airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/schemas/lists.json create mode 100644 airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/source.py create mode 100644 airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/spec.yaml create mode 100644 docs/integrations/sources/emailoctopus.md diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 288a1d584ee6a..ee98e6bcba5dd 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -354,6 +354,13 @@ icon: airbyte.svg sourceType: api releaseStage: alpha +- name: EmailOctopus + sourceDefinitionId: 46b25e70-c980-4590-a811-8deaf50ee09f + dockerRepository: airbyte/source-emailoctopus + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/emailoctopus + sourceType: api + releaseStage: alpha - name: Exchange Rates Api sourceDefinitionId: e2b40e36-aa0e-4bed-b41b-bcea6fa348b1 dockerRepository: airbyte/source-exchange-rates diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index cc66436551f29..b16333418db1c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3021,6 +3021,26 @@ supportsDBT: false supported_destination_sync_modes: [] protocol_version: "0.2.1" +- dockerImage: "airbyte/source-emailoctopus:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/emailoctopus" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "EmailOctopus Spec" + type: "object" + required: + - "api_key" + additionalProperties: true + properties: + api_key: + type: "string" + title: "EmailOctopus API key" + description: "EmailOctopus API Key. See the docs for information on how to generate this key." + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-exchange-rates:1.2.7" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/exchangeratesapi" diff --git a/airbyte-integrations/connectors/source-emailoctopus/.dockerignore b/airbyte-integrations/connectors/source-emailoctopus/.dockerignore new file mode 100644 index 0000000000000..bcd24bdbb33b9 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_emailoctopus +!setup.py +!secrets \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-emailoctopus/BOOTSTRAP.md b/airbyte-integrations/connectors/source-emailoctopus/BOOTSTRAP.md new file mode 100644 index 0000000000000..235e73bf8af08 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/BOOTSTRAP.md @@ -0,0 +1,8 @@ +# EmailOctopus + +EmailOctopus is an email marketing tool. Link to API [here](https://emailoctopus.com/api-documentation). + +## How to get an API key +- [Sign up for EmailOctopus](https://emailoctopus.com/account/sign-up). I recall there is a verification process that involves speaking with support staff. +- Pricing is volume-based, so a sandbox account should be free: see [Pricing](https://emailoctopus.com/pricing). +- Once signed in, generate an API key from the [API documentation page](https://emailoctopus.com/api-documentation). \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-emailoctopus/Dockerfile b/airbyte-integrations/connectors/source-emailoctopus/Dockerfile new file mode 100644 index 0000000000000..3e727c3ebb2ba --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_emailoctopus ./source_emailoctopus + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-emailoctopus \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-emailoctopus/README.md b/airbyte-integrations/connectors/source-emailoctopus/README.md new file mode 100644 index 0000000000000..0c850c27fa3c3 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/README.md @@ -0,0 +1,79 @@ +# EmailOctopus Source + +This is the repository for the EmailOctopus configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/emailoctopus). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-emailoctopus:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/emailoctopus) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_emailoctopus/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source emailoctopus test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-emailoctopus:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-emailoctopus:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-emailoctopus:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-emailoctopus:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-emailoctopus:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-emailoctopus:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-emailoctopus:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-emailoctopus:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-emailoctopus/__init__.py b/airbyte-integrations/connectors/source-emailoctopus/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-config.yml b/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-config.yml new file mode 100644 index 0000000000000..fa2b66fee5df8 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-config.yml @@ -0,0 +1,27 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-emailoctopus:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_emailoctopus/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + bypass_reason: "This connector does not implement incremental sync" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-docker.sh new file mode 100755 index 0000000000000..fa680528f222c --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-docker.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-emailoctopus/build.gradle b/airbyte-integrations/connectors/source-emailoctopus/build.gradle new file mode 100644 index 0000000000000..e9b31b83826ac --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_emailoctopus' +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-emailoctopus/integration_tests/__init__.py b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-emailoctopus/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/acceptance.py new file mode 100644 index 0000000000000..e2a8f1a4cb74d --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + pass diff --git a/airbyte-integrations/connectors/source-emailoctopus/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..c874cae945bb2 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/configured_catalog.json @@ -0,0 +1,34 @@ +{ + "streams": [ + { + "cursor_field": null, + "destination_sync_mode": "overwrite", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "campaigns", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "overwrite", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "lists", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + } + ] +} diff --git a/airbyte-integrations/connectors/source-emailoctopus/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..dc521ade7acf9 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "" +} diff --git a/airbyte-integrations/connectors/source-emailoctopus/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/sample_config.json new file mode 100644 index 0000000000000..013e22132796e --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "your_api_key" +} diff --git a/airbyte-integrations/connectors/source-emailoctopus/main.py b/airbyte-integrations/connectors/source-emailoctopus/main.py new file mode 100644 index 0000000000000..4a250c9ca57cc --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_emailoctopus import SourceEmailoctopus + +if __name__ == "__main__": + source = SourceEmailoctopus() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-emailoctopus/requirements.txt b/airbyte-integrations/connectors/source-emailoctopus/requirements.txt new file mode 100644 index 0000000000000..78140e52009f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-emailoctopus/setup.py b/airbyte-integrations/connectors/source-emailoctopus/setup.py new file mode 100644 index 0000000000000..8d763ab4793da --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_emailoctopus", + description="Source implementation for Emailoctopus.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/__init__.py b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/__init__.py new file mode 100644 index 0000000000000..a548e3428f31c --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceEmailoctopus + +__all__ = ["SourceEmailoctopus"] diff --git a/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/emailoctopus.yaml b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/emailoctopus.yaml new file mode 100644 index 0000000000000..a613bb76c4c71 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/emailoctopus.yaml @@ -0,0 +1,76 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: ["data"] + requester: + url_base: "https://emailoctopus.com/api/1.6" + http_method: "GET" + request_options_provider: + request_parameters: + api_key: "{{ config['api_key'] }}" + increment_paginator: + type: DefaultPaginator + url_base: "*ref(definitions.requester.url_base)" + pagination_strategy: + type: PageIncrement + page_size: 50 + page_size_option: + inject_into: "request_parameter" + field_name: "limit" + page_token_option: + inject_into: "request_parameter" + field_name: "page" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + # API Docs: https://emailoctopus.com/api-documentation/campaigns/get-all + campaigns_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "campaigns" + primary_key: "id" + path: "/campaigns" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + # Fields are large and may affect performance + transformations: + - type: RemoveFields + field_pointers: + - ["content", "html"] + - ["content", "plain_text"] + # API Docs: https://emailoctopus.com/api-documentation/lists/get-all + lists_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "lists" + primary_key: "id" + path: "/lists" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + # 'tags' array not yet documented (2022-10-29) + transformations: + - type: RemoveFields + field_pointers: + - ["tags"] + +streams: + - "*ref(definitions.campaigns_stream)" + - "*ref(definitions.lists_stream)" + +check: + stream_names: + - "campaigns" + - "lists" diff --git a/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/schemas/campaigns.json b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/schemas/campaigns.json new file mode 100644 index 0000000000000..f2da8ae7e58e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/schemas/campaigns.json @@ -0,0 +1,62 @@ +{ + "type": "object", + "title": "Campaigns", + "description": "Details of all campaigns.", + "properties": { + "id": { + "type": "string", + "title": "Campaign ID", + "description": "The identifier of the campaign." + }, + "status": { + "type": "string", + "title": "Status", + "description": "The status of the campaign (DRAFT/SENDING/SENT/ERROR)." + }, + "name": { + "type": ["string", "null"], + "title": "Campaign Name", + "description": "The name of the campaign." + }, + "subject": { + "type": ["string", "null"], + "title": "Subject", + "description": "The subject of the campaign." + }, + "to": { + "type": "array", + "title": "To", + "description": "The ids of the lists the campaign was sent to.", + "items": { + "type": "string" + } + }, + "from": { + "type": "object", + "title": "From", + "description": "The sender of the campaign.", + "properties": { + "name": { + "type": "string", + "title": "Sender Name", + "description": "The name the campaign was sent from." + }, + "email_address": { + "type": "string", + "title": "Sender Email Address", + "description": "The email address the campaign was sent from." + } + } + }, + "created_at": { + "type": "string", + "title": "Campaign Created At", + "description": "When the campaign was created, in ISO 8601 format." + }, + "sent_at": { + "type": "string", + "title": "Campaign Sent At", + "description": "When the campaign was sent, in ISO 8601 format." + } + } +} diff --git a/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/schemas/lists.json b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/schemas/lists.json new file mode 100644 index 0000000000000..d3ce7ad20cba7 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/schemas/lists.json @@ -0,0 +1,76 @@ +{ + "type": "object", + "title": "Lists", + "description": "Details of all lists.", + "properties": { + "id": { + "type": "string", + "title": "List ID", + "description": "The identifier of the list." + }, + "name": { + "type": "string", + "title": "List Name", + "description": "The name of the list." + }, + "double_opt_in": { + "type": "boolean", + "title": "Double Opt-in", + "description": "If double opt-in has been enabled on the list." + }, + "fields": { + "type": "array", + "title": "Subscriber Fields", + "description": "Stored information on subscribers.", + "properties": { + "tag": { + "type": "string", + "title": "Tag", + "description": "The identifier used to reference the field in your emails." + }, + "type": { + "type": "string", + "title": "Type", + "description": "The type of the field - can be NUMBER, TEXT or DATE." + }, + "label": { + "type": "string", + "title": "Label", + "description": "A human readable label for the field." + }, + "fallback": { + "type": "null", + "title": "Fallback", + "description": "A default value for the field, used when there is no other value available." + } + } + }, + "counts": { + "type": "object", + "title": "Counts", + "description": "The summary counts of the list.", + "properties": { + "pending": { + "type": "integer", + "title": "Pending", + "description": "The number of pending contacts in the list." + }, + "subscribed": { + "type": "integer", + "title": "Subscribed", + "description": "The number of subscribed contacts in the list." + }, + "unsubscribed": { + "type": "integer", + "title": "Unsubscribed", + "description": "The number of unsubscribed contacts in the list." + } + } + }, + "created_at": { + "type": "string", + "title": "List Created At", + "description": "When the list was created, in ISO 8601 format." + } + } +} diff --git a/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/source.py b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/source.py new file mode 100644 index 0000000000000..738ef2970b88b --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceEmailoctopus(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "emailoctopus.yaml"}) diff --git a/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/spec.yaml b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/spec.yaml new file mode 100644 index 0000000000000..75da7a9bbd392 --- /dev/null +++ b/airbyte-integrations/connectors/source-emailoctopus/source_emailoctopus/spec.yaml @@ -0,0 +1,17 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/emailoctopus +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: EmailOctopus Spec + type: object + required: + - api_key + additionalProperties: true + properties: + api_key: + type: string + title: EmailOctopus API key + description: >- + EmailOctopus API Key. See the docs + for information on how to generate this key. + airbyte_secret: true diff --git a/docs/integrations/sources/emailoctopus.md b/docs/integrations/sources/emailoctopus.md new file mode 100644 index 0000000000000..5ac424be176ac --- /dev/null +++ b/docs/integrations/sources/emailoctopus.md @@ -0,0 +1,27 @@ +# EmailOctopus + +## Requirements +* [EmailOctopus account](https://help.emailoctopus.com) +* EmailOctopus [API key](https://help.emailoctopus.com/article/165-how-to-create-and-delete-api-keys) + +## Supported sync modes + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | [Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite) | +| Incremental Sync | No | | + +## Supported Streams + +* [Get all campaigns](https://emailoctopus.com/api-documentation/campaigns/get-all) +* [Get all lists](https://emailoctopus.com/api-documentation/lists/get-all) + +## Performance considerations + +No documented strict rate limit. + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :----------- |:-----------------------------------------------------------| +| 0.1.0 | 2022-10-29 | [18647](https://github.com/airbytehq/airbyte/pull/18647) | Initial commit | \ No newline at end of file