From 029916bbe1d860bd316108fed24d7720d8be3494 Mon Sep 17 00:00:00 2001 From: Johan Hammarstedt Date: Tue, 1 Nov 2022 23:03:41 +0100 Subject: [PATCH 01/10] added WIP connector Postmarkapp --- .../resources/seed/source_definitions.yaml | 8 ++ .../source_delighted/source.py | 2 +- .../source-postmarkapp/.dockerignore | 6 ++ .../connectors/source-postmarkapp/Dockerfile | 38 ++++++++ .../connectors/source-postmarkapp/README.md | 79 +++++++++++++++++ .../connectors/source-postmarkapp/__init__.py | 3 + .../acceptance-test-config.yml | 38 ++++++++ .../acceptance-test-docker.sh | 16 ++++ .../source-postmarkapp/build.gradle | 9 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 ++ .../integration_tests/acceptance.py | 16 ++++ .../integration_tests/configured_catalog.json | 35 ++++++++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 ++ .../connectors/source-postmarkapp/main.py | 13 +++ .../source-postmarkapp/requirements.txt | 2 + .../connectors/source-postmarkapp/setup.py | 29 +++++++ .../source_postmarkapp/__init__.py | 8 ++ .../source_postmarkapp/postmarkapp.yaml | 86 +++++++++++++++++++ .../source_postmarkapp/schemas/bounces.json | 65 ++++++++++++++ .../schemas/deliverystats.json | 24 ++++++ .../schemas/message-streams.json | 41 +++++++++ .../source_postmarkapp/schemas/messages.json | 83 ++++++++++++++++++ .../schemas/outbound-stats.json | 57 ++++++++++++ .../source_postmarkapp/schemas/server.json | 76 ++++++++++++++++ .../source_postmarkapp/schemas/servers.json | 77 +++++++++++++++++ .../source_postmarkapp/schemas/templates.json | 32 +++++++ .../source_postmarkapp/source.py | 18 ++++ .../source_postmarkapp/spec.yaml | 14 +++ docs/integrations/sources/postmarkapp.md | 60 +++++++++++++ 32 files changed, 953 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-postmarkapp/.dockerignore create mode 100644 airbyte-integrations/connectors/source-postmarkapp/Dockerfile create mode 100644 airbyte-integrations/connectors/source-postmarkapp/README.md create mode 100644 airbyte-integrations/connectors/source-postmarkapp/__init__.py create mode 100644 airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-postmarkapp/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-postmarkapp/build.gradle create mode 100644 airbyte-integrations/connectors/source-postmarkapp/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-postmarkapp/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/main.py create mode 100644 airbyte-integrations/connectors/source-postmarkapp/requirements.txt create mode 100644 airbyte-integrations/connectors/source-postmarkapp/setup.py create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/__init__.py create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/bounces.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/deliverystats.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/message-streams.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/messages.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/outbound-stats.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/server.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/templates.json create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/source.py create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/spec.yaml create mode 100644 docs/integrations/sources/postmarkapp.md diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 78d58ab5f19c4..254a41cb4a7d4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1075,6 +1075,14 @@ icon: postgresql.svg sourceType: database releaseStage: generally_available +- name: Postmarkapp + sourceDefinitionId: cde75ca1-1e28-4a0f-85bb-90c546de9f1f + dockerRepository: airbyte/source-postmarkapp + dockerImageTag: 0.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/postmarkapp + icon: postmarkapp.svg + sourceType: api + releaseStage: alpha - name: Prestashop sourceDefinitionId: d60a46d4-709f-4092-a6b7-2457f7d455f5 dockerRepository: airbyte/source-prestashop diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index f584d5c987920..e4f2583db75dc 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -118,7 +118,7 @@ class Bounces(IncrementalDelightedStream): primary_key = "person_id" def path(self, **kwargs) -> str: - return "bounces.json" + return "deliverystats.json" class SurveyResponses(IncrementalDelightedStream): diff --git a/airbyte-integrations/connectors/source-postmarkapp/.dockerignore b/airbyte-integrations/connectors/source-postmarkapp/.dockerignore new file mode 100644 index 0000000000000..5122f90d02056 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_postmarkapp +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-postmarkapp/Dockerfile b/airbyte-integrations/connectors/source-postmarkapp/Dockerfile new file mode 100644 index 0000000000000..c9662a9ddb090 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_postmarkapp ./source_postmarkapp + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-postmarkapp diff --git a/airbyte-integrations/connectors/source-postmarkapp/README.md b/airbyte-integrations/connectors/source-postmarkapp/README.md new file mode 100644 index 0000000000000..c44654ce8ddf1 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/README.md @@ -0,0 +1,79 @@ +# Postmarkapp Source + +This is the repository for the Postmarkapp configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/postmarkapp). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-postmarkapp:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/postmarkapp) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_postmarkapp/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source postmarkapp test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-postmarkapp:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-postmarkapp:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-postmarkapp:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-postmarkapp:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-postmarkapp:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-postmarkapp:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-postmarkapp:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-postmarkapp:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-postmarkapp/__init__.py b/airbyte-integrations/connectors/source-postmarkapp/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml new file mode 100644 index 0000000000000..02cb7f40e471e --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml @@ -0,0 +1,38 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-postmarkapp:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_postmarkapp/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-postmarkapp/build.gradle b/airbyte-integrations/connectors/source-postmarkapp/build.gradle new file mode 100644 index 0000000000000..cb04abfc4ae72 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_postmarkapp' +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/__init__.py b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..52b0f2c2118f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/acceptance.py new file mode 100644 index 0000000000000..1302b2f57e10e --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..6384cb09032bb --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json @@ -0,0 +1,35 @@ +{ + "streams": [ + { + "stream": { + "name": "deliverystats", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream" : { + "name" : "message-streams", + "json_schema" : {}, + "supported_sync_modes" : [ + "full_refresh" + ] + }, + "sync_mode" : "full_refresh", + "destination_sync_mode" : "overwrite" + }, + { + "stream" : { + "name" : "server", + "json_schema" : {}, + "supported_sync_modes" : [ + "full_refresh" + ] + }, + "sync_mode" : "full_refresh", + "destination_sync_mode" : "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..4ae0ceadeae69 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "X-Postmark-Server-Token": "wrong_value" +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/sample_config.json new file mode 100644 index 0000000000000..3efed19b653ec --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "X-Postmark-Server-Token": "test8616-1337-44ee-te33-235467d79901b" +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/sample_state.json new file mode 100644 index 0000000000000..22f8fcf41fe91 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "message-streams": { + "ServerID": 12 + } +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/main.py b/airbyte-integrations/connectors/source-postmarkapp/main.py new file mode 100644 index 0000000000000..937988c5c3932 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_postmarkapp import SourcePostmarkapp + +if __name__ == "__main__": + source = SourcePostmarkapp() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-postmarkapp/requirements.txt b/airbyte-integrations/connectors/source-postmarkapp/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-postmarkapp/setup.py b/airbyte-integrations/connectors/source-postmarkapp/setup.py new file mode 100644 index 0000000000000..e285eb5e672fa --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_postmarkapp", + description="Source implementation for Postmarkapp.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/__init__.py b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/__init__.py new file mode 100644 index 0000000000000..0ecd9826b1a2b --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourcePostmarkapp + +__all__ = ["SourcePostmarkapp"] diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml new file mode 100644 index 0000000000000..52673f27ee10b --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml @@ -0,0 +1,86 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: [] + requester: + url_base: "https://api.postmarkapp.com" + http_method: "GET" + request_options_provider: + request_body_json: + Accept: "application/json" + authenticator: + type: "ApiKeyAuthenticator" + header: "X-Postmark-Server-Token" + api_token: "{{ config['X-Postmark-Server-Token'] }}" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + offset_paginator: + type: DefaultPaginator + $options: + url_base: "*ref(definitions.requester.url_base)" + pagination_strategy: + type: "OffsetIncrement" + page_size: 500 + page_token_option: + inject_into: "request_parameter" + field_name: "offset" + page_size_option: + inject_into: "request_parameter" + field_name: "count" + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + deliverystats: + $ref: "*ref(definitions.base_stream)" + $options: + name: "deliverystats" + primary_key: "Type" + path: "/deliverystats" + message-streams: + $ref: "*ref(definitions.base_stream)" + $options: + name: "message-streams" + primary_key: "ID" + path: "/message-streams" + bounces: + $ref: "*ref(definitions.base_stream)" + $options: + name: "bounces" + primary_key: "ID" + path: "/bounces" + server: + $ref: "*ref(definitions.base_stream)" + $options: + name: "server" + primary_key: "ID" + path: "/server" + servers: + $ref: "*ref(definitions.base_stream)" + $options: + name: "servers" + primary_key: "ID" + path: "/servers" + messages: + $ref: "*ref(definitions.base_stream)" + $options: + name: "messages" + primary_key: "MessageID" + path: "/messages/outbound" + +streams: + - "*ref(definitions.deliverystats)" + - "*ref(definitions.message-streams)" + - "*ref(definitions.server)" + #- "*ref(definitions.messages)" + #- "*ref(definitions.outbound-stats)" + #- "*ref(definitions.bounces)" + + +check: + stream_names: + - "deliverystats" diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/bounces.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/bounces.json new file mode 100644 index 0000000000000..0b59f7ec823d5 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/bounces.json @@ -0,0 +1,65 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "TotalCount": { + "type": "integer" + }, + "Bounces": { + "type": "object", + "properties": { + "ID": { + "type": "integer" + }, + "Type": { + "type": "string" + }, + "TypeCode": { + "type": "integer" + }, + "Name": { + "type": "string" + }, + "Tag": { + "type": "string" + }, + "MessageID": { + "type": "string" + }, + "ServerID": { + "type": "integer" + }, + "MessageStream": { + "type": "string" + }, + "Description": { + "type": "string" + }, + "Details": { + "type": "string" + }, + "Email": { + "type": "string" + }, + "From": { + "type": "string" + }, + "BouncedAt": { + "type": "string" + }, + "DumpAvailable": { + "type": "boolean" + }, + "Inactive": { + "type": "boolean" + }, + "CanActivate": { + "type": "boolean" + }, + "Subject": { + "type": "string" + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/deliverystats.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/deliverystats.json new file mode 100644 index 0000000000000..ff3d6fffbcb54 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/deliverystats.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "InactiveMails": { + "type": "integer" + }, + "Bounces": { + "type": "array", + "items": + { + "type": "object", + "properties": { + "Name": { + "type": "string" + }, + "Count": { + "type": "integer" + } + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/message-streams.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/message-streams.json new file mode 100644 index 0000000000000..c79c10bc85113 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/message-streams.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "ID": { + "type": "string" + }, + "ServerID": { + "type": "integer" + }, + "Name": { + "type": "string" + }, + "Description": { + "type": "string" + }, + "MessageStreamType": { + "type": "string" + }, + "CreatedAt": { + "type": "string" + }, + "UpdatedAt": { + "type": "null" + }, + "ArchivedAt": { + "type": "null" + }, + "ExpectedPurgeDate": { + "type": "null" + }, + "SubscriptionManagementConfiguration": { + "type": "object", + "properties": { + "UnsubscribeHandlingType": { + "type": "string" + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/messages.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/messages.json new file mode 100644 index 0000000000000..e2e6c21b51f77 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/messages.json @@ -0,0 +1,83 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "Tag": { + "type": "string" + }, + "MessageID": { + "type": "string" + }, + "MessageStream": { + "type": "string" + }, + "To": { + "type": "array", + "items": + { + "type": "object", + "properties": { + "Email": { + "type": "string" + }, + "Name": { + "type": "null" + } + } + } + + }, + "Cc": { + "type": "array", + "items": {} + }, + "Bcc": { + "type": "array", + "items": {} + }, + "Recipients": { + "type": "array", + "items": [ + { + "type": "string" + } + ] + }, + "ReceivedAt": { + "type": "string" + }, + "From": { + "type": "string" + }, + "Subject": { + "type": "string" + }, + "Attachments": { + "type": "array", + "items": {} + }, + "Status": { + "type": "string" + }, + "TrackOpens": { + "type": "boolean" + }, + "TrackLinks": { + "type": "string" + }, + "Metadata": { + "type": "object", + "properties": { + "color": { + "type": "string" + }, + "client-id": { + "type": "string" + } + } + }, + "Sandboxed": { + "type": "boolean" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/outbound-stats.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/outbound-stats.json new file mode 100644 index 0000000000000..b05e387519339 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/outbound-stats.json @@ -0,0 +1,57 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "Sent": { + "type": "integer" + }, + "Bounced": { + "type": "integer" + }, + "SMTPApiErrors": { + "type": "integer" + }, + "BounceRate": { + "type": "number" + }, + "SpamComplaints": { + "type": "integer" + }, + "SpamComplaintsRate": { + "type": "number" + }, + "Tracked": { + "type": "integer" + }, + "Opens": { + "type": "integer" + }, + "UniqueOpens": { + "type": "integer" + }, + "TotalClicks": { + "type": "integer" + }, + "UniqueLinksClicked": { + "type": "integer" + }, + "WithClientRecorded": { + "type": "integer" + }, + "WithPlatformRecorded": { + "type": "integer" + }, + "WithReadTimeRecorded": { + "type": "integer" + }, + "WithLinkTracking": { + "type": "integer" + }, + "WithOpenTracking": { + "type": "integer" + }, + "TotalTrackedLinksSent": { + "type": "integer" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/server.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/server.json new file mode 100644 index 0000000000000..5d3ac676065c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/server.json @@ -0,0 +1,76 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "ID": { + "type": "integer" + }, + "Name": { + "type": "string" + }, + "ApiTokens": { + "type": "array", + "items": + { + "type": "string" + } + }, + "Color": { + "type": "string" + }, + "SmtpApiActivated": { + "type": "boolean" + }, + "RawEmailEnabled": { + "type": "boolean" + }, + "DeliveryType": { + "type": "string" + }, + "ServerLink": { + "type": "string" + }, + "InboundAddress": { + "type": "string" + }, + "InboundHookUrl": { + "type": "string" + }, + "BounceHookUrl": { + "type": "string" + }, + "OpenHookUrl": { + "type": "string" + }, + "DeliveryHookUrl": { + "type": "string" + }, + "PostFirstOpenOnly": { + "type": "boolean" + }, + "InboundDomain": { + "type": "string" + }, + "InboundHash": { + "type": "string" + }, + "InboundSpamThreshold": { + "type": "integer" + }, + "TrackOpens": { + "type": "boolean" + }, + "TrackLinks": { + "type": "string" + }, + "IncludeBounceContentInHook": { + "type": "boolean" + }, + "ClickHookUrl": { + "type": "string" + }, + "EnableSmtpApiErrorHooks": { + "type": "boolean" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json new file mode 100644 index 0000000000000..f642ec2d3fc74 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json @@ -0,0 +1,77 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "ID": { + "type": "integer" + }, + "Name": { + "type": "string" + }, + "ApiTokens": { + "type": "array", + "items": [ + { + "type": "string" + } + ] + }, + "Color": { + "type": "string" + }, + "SmtpApiActivated": { + "type": "boolean" + }, + "RawEmailEnabled": { + "type": "boolean" + }, + "DeliveryType": { + "type": "string" + }, + "ServerLink": { + "type": "string" + }, + "InboundAddress": { + "type": "string" + }, + "InboundHookUrl": { + "type": "string" + }, + "BounceHookUrl": { + "type": "string" + }, + "OpenHookUrl": { + "type": "string" + }, + "DeliveryHookUrl": { + "type": "string" + }, + "PostFirstOpenOnly": { + "type": "boolean" + }, + "InboundDomain": { + "type": "string" + }, + "InboundHash": { + "type": "string" + }, + "InboundSpamThreshold": { + "type": "integer" + }, + "TrackOpens": { + "type": "boolean" + }, + "TrackLinks": { + "type": "string" + }, + "IncludeBounceContentInHook": { + "type": "boolean" + }, + "ClickHookUrl": { + "type": "string" + }, + "EnableSmtpApiErrorHooks": { + "type": "boolean" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/templates.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/templates.json new file mode 100644 index 0000000000000..fb5a8f3809f58 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/templates.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "TotalCount": { + "type": "integer" + }, + "Templates": { + "type": "object", + "properties": { + "Active": { + "type": "boolean" + }, + "TemplateId": { + "type": "integer" + }, + "Name": { + "type": "string" + }, + "Alias": { + "type": "string" + }, + "TemplateType": { + "type": "string" + }, + "LayoutTemplate": { + "type": "string" + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/source.py b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/source.py new file mode 100644 index 0000000000000..1b110964ac1db --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourcePostmarkapp(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "postmarkapp.yaml"}) diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/spec.yaml b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/spec.yaml new file mode 100644 index 0000000000000..d42e2dd308f66 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/spec.yaml @@ -0,0 +1,14 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Postmarkapp Spec + type: object + required: + - X-Postmark-Server-Token + additionalProperties: true + properties: + X-Postmark-Server-Token: + title: X-Postmark-Server-Token + type: string + description: API Key + airbyte_secret: true diff --git a/docs/integrations/sources/postmarkapp.md b/docs/integrations/sources/postmarkapp.md new file mode 100644 index 0000000000000..0cb1567433abf --- /dev/null +++ b/docs/integrations/sources/postmarkapp.md @@ -0,0 +1,60 @@ +# Postmarkapp + +## Overview + +The Postmarkapp source can sync data from the [Postmarkapp API](https://postmarkapp.com/developer) + +## Requirements + +Postmarkapp requires an API key to make request and retrieve data. You can find your API key in the [Postmarkapp dashboard](https://account.postmarkapp.com/servers/9708911/credentials). + +## Streams +Current supported streams: + +Server-API +- [Bounces: Deliverystats](https://postmarkapp.com/developer/api/bounce-api#delivery-stats): Lets you access all reports regarding your bounces for a specific server. Bounces are available for 45 days after a bounce. +- [Message-Streams](https://postmarkapp.com/developer/api/message-streams-api#list-message-streams): Lets you manage message streams for a specific server. Please note: A Server may have up to 10 Streams, including the default ones. Default Streams cannot be deleted, and Servers can only have 1 Inbound Stream. +- [Outbound stats](https://account.postmarkapp.com/servers/9708911/credentials): Lets you get all of the statistics of your outbound emails for a specific server. These statistics are stored permantently and do not expire. All stats use EST timezone + +TODO: +Account-API +- [Servers](https://postmarkapp.com/developer/api/servers-api): Lets you manage servers for a specific account. +- [Domains](https://postmarkapp.com/developer/api/domains-api): Gets a list of domains containing an overview of the domain and authentication status. +- [Sender signatures](https://postmarkapp.com/developer/api/signatures-api): Gets a list of sender signatures containing brief details associated with your account. + + + + +## Setup guide + +## Step 1: Set up the Postmarkapp connector in Airbyte + +### For Airbyte Cloud: + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. +3. On the Set up the source page, select **Postmarkapp** from the Source type dropdown. +4. Click **Set up source**. + +### For Airbyte OSS: + +1. Navigate to the Airbyte Open Source dashboard. +2. Set the name for your source (Postmarkapp). +3. Click **Set up source**. + +## Supported sync modes + +The Postmarkapp source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +| Feature | Supported? | +| :---------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| Namespaces | No | + + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ |:-----|:-------------| :----------------------------------------- | +| 0.1.0 | TBA | TBA | 🎉 New Source: Postmarkapp API [low-code CDK] | \ No newline at end of file From f5d1bcb694e629bb9ea88b085646e1be0d6a27b0 Mon Sep 17 00:00:00 2001 From: Johan Hammarstedt Date: Wed, 2 Nov 2022 21:46:13 +0100 Subject: [PATCH 02/10] pagination fix --- .../acceptance-test-config.yml | 11 +--- .../acceptance-test-docker.sh | 0 .../integration_tests/configured_catalog.json | 35 +++++++++- .../connectors/source-postmarkapp/setup.py | 2 +- .../source_postmarkapp/postmarkapp.yaml | 64 ++++++++++++++----- .../source_postmarkapp/schemas/domains.json | 37 +++++++++++ .../source_postmarkapp/schemas/messages.json | 4 +- .../source_postmarkapp/schemas/servers.json | 11 +++- .../source_postmarkapp/spec.yaml | 8 ++- 9 files changed, 142 insertions(+), 30 deletions(-) mode change 100644 => 100755 airbyte-integrations/connectors/source-postmarkapp/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/domains.json diff --git a/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml index 02cb7f40e471e..2f11ee2462f61 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml @@ -18,20 +18,15 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + empty_streams: + - name: bounces + bypass_reason: "The bounces stream" # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: # path: "integration_tests/expected_records.txt" # extra_fields: no # exact_order: no # extra_records: yes - incremental: - bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-docker.sh old mode 100644 new mode 100755 diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json index 6384cb09032bb..809ea47eeea42 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json @@ -22,7 +22,40 @@ }, { "stream" : { - "name" : "server", + "name" : "servers", + "json_schema" : {}, + "supported_sync_modes" : [ + "full_refresh" + ] + }, + "sync_mode" : "full_refresh", + "destination_sync_mode" : "overwrite" + }, + { + "stream" : { + "name" : "messages", + "json_schema" : {}, + "supported_sync_modes" : [ + "full_refresh" + ] + }, + "sync_mode" : "full_refresh", + "destination_sync_mode" : "overwrite" + }, + { + "stream" : { + "name" : "domains", + "json_schema" : {}, + "supported_sync_modes" : [ + "full_refresh" + ] + }, + "sync_mode" : "full_refresh", + "destination_sync_mode" : "overwrite" + }, + { + "stream" : { + "name" : "bounces", "json_schema" : {}, "supported_sync_modes" : [ "full_refresh" diff --git a/airbyte-integrations/connectors/source-postmarkapp/setup.py b/airbyte-integrations/connectors/source-postmarkapp/setup.py index e285eb5e672fa..a29c3ebdad3d7 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/setup.py +++ b/airbyte-integrations/connectors/source-postmarkapp/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk~=0.4", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml index 52673f27ee10b..4a93da15ca1cd 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml @@ -10,14 +10,46 @@ definitions: request_options_provider: request_body_json: Accept: "application/json" + request_parameters: + count: '500' authenticator: type: "ApiKeyAuthenticator" header: "X-Postmark-Server-Token" api_token: "{{ config['X-Postmark-Server-Token'] }}" + requester_account: + url_base: "https://api.postmarkapp.com" + http_method: "GET" + request_options_provider: + request_body_json: + Accept: "application/json" + request_parameters: + count: '500' + authenticator: + type: "ApiKeyAuthenticator" + header: "X-Postmark-Account-Token" + api_token: "{{ config['X-Postmark-Account-Token'] }}" + retriever_account: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: DefaultPaginator + $options: + url_base: "*ref(definitions.requester.url_base)" + pagination_strategy: + type: "OffsetIncrement" + page_size: 500 + page_token_option: + inject_into: "request_parameter" + field_name: "offset" + page_size_option: + inject_into: "request_parameter" + field_name: "offset" + requester: + $ref: "*ref(definitions.requester_account)" retriever: record_selector: $ref: "*ref(definitions.selector)" - offset_paginator: + paginator: type: DefaultPaginator $options: url_base: "*ref(definitions.requester.url_base)" @@ -29,12 +61,15 @@ definitions: field_name: "offset" page_size_option: inject_into: "request_parameter" - field_name: "count" + field_name: "offset" requester: $ref: "*ref(definitions.requester)" base_stream: retriever: $ref: "*ref(definitions.retriever)" + base_stream_account: + retriever: + $ref: "*ref(definitions.retriever_account)" deliverystats: $ref: "*ref(definitions.base_stream)" $options: @@ -53,14 +88,8 @@ definitions: name: "bounces" primary_key: "ID" path: "/bounces" - server: - $ref: "*ref(definitions.base_stream)" - $options: - name: "server" - primary_key: "ID" - path: "/server" servers: - $ref: "*ref(definitions.base_stream)" + $ref: "*ref(definitions.base_stream_account)" $options: name: "servers" primary_key: "ID" @@ -71,16 +100,21 @@ definitions: name: "messages" primary_key: "MessageID" path: "/messages/outbound" - + domains: + $ref: "*ref(definitions.base_stream_account)" + $options: + name: "domains" + primary_key: "ID" + path: "/domains" streams: - "*ref(definitions.deliverystats)" - "*ref(definitions.message-streams)" - - "*ref(definitions.server)" - #- "*ref(definitions.messages)" - #- "*ref(definitions.outbound-stats)" - #- "*ref(definitions.bounces)" + - "*ref(definitions.domains)" + - "*ref(definitions.messages)" + - "*ref(definitions.bounces)" + - "*ref(definitions.servers)" check: stream_names: - - "deliverystats" + - "servers" diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/domains.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/domains.json new file mode 100644 index 0000000000000..3cf12c51da9e7 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/domains.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "TotalCount": { + "type": "integer" + }, + "Domains": { + "type": "array", + "items": + { + "type": "object", + "properties": { + "Name": { + "type": "string" + }, + "SPFVerified": { + "type": "boolean" + }, + "DKIMVerified": { + "type": "boolean" + }, + "WeakDKIM": { + "type": "boolean" + }, + "ReturnPathDomainVerified": { + "type": "boolean" + }, + "ID": { + "type": "integer" + } + } + } + + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/messages.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/messages.json index e2e6c21b51f77..98807322b1556 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/messages.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/messages.json @@ -37,11 +37,11 @@ }, "Recipients": { "type": "array", - "items": [ + "items": { "type": "string" } - ] + }, "ReceivedAt": { "type": "string" diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json index f642ec2d3fc74..baa31350a029a 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json @@ -2,6 +2,12 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { + "TotalCount": { + "type": "integer" + }, + "Servers": { + "type": "array", + "items": { "ID": { "type": "integer" }, @@ -10,11 +16,10 @@ }, "ApiTokens": { "type": "array", - "items": [ + "items": { "type": "string" } - ] }, "Color": { "type": "string" @@ -73,5 +78,7 @@ "EnableSmtpApiErrorHooks": { "type": "boolean" } + } + } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/spec.yaml b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/spec.yaml index d42e2dd308f66..d55d17e66572e 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/spec.yaml +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/spec.yaml @@ -5,10 +5,16 @@ connectionSpecification: type: object required: - X-Postmark-Server-Token + - X-Postmark-Account-Token additionalProperties: true properties: X-Postmark-Server-Token: title: X-Postmark-Server-Token type: string - description: API Key + description: API Key for server airbyte_secret: true + X-Postmark-Account-Token: + title: X-Postmark-Account-Token + type: string + description: API Key for account + airbyte_secret: true From 1ddbe19c4e99f5140f8c0d072b9b71bbccbabeb4 Mon Sep 17 00:00:00 2001 From: Johan Hammarstedt Date: Wed, 2 Nov 2022 21:55:21 +0100 Subject: [PATCH 03/10] fixing tests --- .../source-postmarkapp/acceptance-test-config.yml | 13 +++++++------ .../integration_tests/expected_records.txt | 6 ++++++ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 airbyte-integrations/connectors/source-postmarkapp/integration_tests/expected_records.txt diff --git a/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml index 2f11ee2462f61..6837feeb35549 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml @@ -21,12 +21,13 @@ acceptance_tests: empty_streams: - name: bounces bypass_reason: "The bounces stream" -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes + - name: message-streams + bypass_reason: "The bounces stream" + expect_records: + path: "integration_tests/expected_records.txt" + extra_fields: no + exact_order: no + extra_records: yes full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/expected_records.txt new file mode 100644 index 0000000000000..915bdae175d30 --- /dev/null +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/expected_records.txt @@ -0,0 +1,6 @@ +{"stream": "deliverystats", "data": {"InactiveMails": 0, "Bounces": [{"Name": "All", "Count": 0}]}, "emitted_at": 1667421988008} +{"stream": "message-streams", "data": {"MessageStreams": [{"ID": "broadcast", "ServerID": 9708911, "Name": "Default Broadcast Stream", "Description": "Default stream used for sending broadcast messages", "MessageStreamType": "Broadcasts", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "Postmark"}}, {"ID": "inbound", "ServerID": 9708911, "Name": "Default Inbound Stream", "Description": "Default stream used for receiving inbound messages", "MessageStreamType": "Inbound", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "None"}}, {"ID": "outbound", "ServerID": 9708911, "Name": "Default Transactional Stream", "Description": "Default stream used for sending transactional messages", "MessageStreamType": "Transactional", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "None"}}], "TotalCount": 3}, "emitted_at": 1667421988591} +{"stream": "servers", "data": {"TotalCount": 1, "Servers": []}, "emitted_at": 1667421989212} +{"stream": "messages", "data": {"TotalCount": 0, "Messages": []}, "emitted_at": 1667421989948} +{"stream": "domains", "data": {"TotalCount": 1, "Domains": []}, "emitted_at": 1667421990520} +{"stream": "bounces", "data": {"TotalCount": 0, "Bounces": []}, "emitted_at": 1667421991204} From b006547430d13939ef042c27f5799158b8dc8b87 Mon Sep 17 00:00:00 2001 From: Johan Hammarstedt Date: Tue, 8 Nov 2022 13:05:27 +0100 Subject: [PATCH 04/10] fixing records --- .../integration_tests/expected_records.txt | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/expected_records.txt index 915bdae175d30..7670bd0095e6b 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/expected_records.txt @@ -1,6 +1,8 @@ -{"stream": "deliverystats", "data": {"InactiveMails": 0, "Bounces": [{"Name": "All", "Count": 0}]}, "emitted_at": 1667421988008} -{"stream": "message-streams", "data": {"MessageStreams": [{"ID": "broadcast", "ServerID": 9708911, "Name": "Default Broadcast Stream", "Description": "Default stream used for sending broadcast messages", "MessageStreamType": "Broadcasts", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "Postmark"}}, {"ID": "inbound", "ServerID": 9708911, "Name": "Default Inbound Stream", "Description": "Default stream used for receiving inbound messages", "MessageStreamType": "Inbound", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "None"}}, {"ID": "outbound", "ServerID": 9708911, "Name": "Default Transactional Stream", "Description": "Default stream used for sending transactional messages", "MessageStreamType": "Transactional", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "None"}}], "TotalCount": 3}, "emitted_at": 1667421988591} -{"stream": "servers", "data": {"TotalCount": 1, "Servers": []}, "emitted_at": 1667421989212} -{"stream": "messages", "data": {"TotalCount": 0, "Messages": []}, "emitted_at": 1667421989948} -{"stream": "domains", "data": {"TotalCount": 1, "Domains": []}, "emitted_at": 1667421990520} -{"stream": "bounces", "data": {"TotalCount": 0, "Bounces": []}, "emitted_at": 1667421991204} +{"stream": "deliverystats", "data": {"InactiveMails": 0, "Bounces": [{"Name": "All", "Count": 0}]}, "emitted_at": 1667895141188} +{"stream": "message-streams", "data": {"MessageStreams": [{"ID": "broadcast", "ServerID": 9708911, "Name": "Default Broadcast Stream", "Description": "Default stream used for sending broadcast messages", "MessageStreamType": "Broadcasts", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "Postmark"}}, {"ID": "inbound", "ServerID": 9708911, "Name": "Default Inbound Stream", "Description": "Default stream used for receiving inbound messages", "MessageStreamType": "Inbound", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "None"}}, {"ID": "outbound", "ServerID": 9708911, "Name": "Default Transactional Stream", "Description": "Default stream used for sending transactional messages", "MessageStreamType": "Transactional", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "None"}}], "TotalCount": 3}, "emitted_at": 1667895141700} +{"stream": "servers", "data": {"TotalCount": 1, "Servers": []}, "emitted_at": 1667895142238} +{"stream": "messages", "data": {"TotalCount": 0, "Messages": []}, "emitted_at": 1667895142930} +{"stream": "domains", "data": {"TotalCount": 1, "Domains": []}, "emitted_at": 1667895143502} +{"stream": "bounces", "data": {"TotalCount": 0, "Bounces": []}, "emitted_at": 1667895144067} + + From 392730973501c7769f82388d8614f2fe13ded19e Mon Sep 17 00:00:00 2001 From: Johan Hammarstedt Date: Wed, 9 Nov 2022 17:56:27 +0100 Subject: [PATCH 05/10] fixing tests --- .../acceptance-test-config.yml | 12 +- .../integration_tests/configured_catalog.json | 8 +- .../integration_tests/expected_records.txt | 12 +- .../source_postmarkapp/postmarkapp.yaml | 45 +++++++- .../source_postmarkapp/schemas/bounces.json | 104 ++++++++---------- .../schemas/deliverystats.json | 20 +--- .../source_postmarkapp/schemas/domains.json | 47 +++----- .../source_postmarkapp/schemas/servers.json | 8 -- 8 files changed, 131 insertions(+), 125 deletions(-) diff --git a/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml index 6837feeb35549..389620d838b46 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-config.yml @@ -20,9 +20,15 @@ acceptance_tests: configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: - name: bounces - bypass_reason: "The bounces stream" - - name: message-streams - bypass_reason: "The bounces stream" + bypass_reason: "Is empty in test data" + - name: messages + bypass_reason: "Is empty in test data" + - name: deliverystats + bypass_reason: "Is empty in test data" + - name: servers + bypass_reason: "Is empty in test data" + - name: domains + bypass_reason: "Is empty in test data" expect_records: path: "integration_tests/expected_records.txt" extra_fields: no diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json index 809ea47eeea42..deec84d6256b7 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json @@ -4,7 +4,8 @@ "stream": { "name": "deliverystats", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["Name"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -13,6 +14,7 @@ "stream" : { "name" : "message-streams", "json_schema" : {}, + "source_defined_primary_key": [["ID"]], "supported_sync_modes" : [ "full_refresh" ] @@ -24,6 +26,7 @@ "stream" : { "name" : "servers", "json_schema" : {}, + "source_defined_primary_key": [["ID"]], "supported_sync_modes" : [ "full_refresh" ] @@ -35,6 +38,7 @@ "stream" : { "name" : "messages", "json_schema" : {}, + "source_defined_primary_key": [["MessageID"]], "supported_sync_modes" : [ "full_refresh" ] @@ -46,6 +50,7 @@ "stream" : { "name" : "domains", "json_schema" : {}, + "source_defined_primary_key": [["ID"]], "supported_sync_modes" : [ "full_refresh" ] @@ -57,6 +62,7 @@ "stream" : { "name" : "bounces", "json_schema" : {}, + "source_defined_primary_key": [["ID"]], "supported_sync_modes" : [ "full_refresh" ] diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/expected_records.txt index 7670bd0095e6b..f54b79d2f917e 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/expected_records.txt @@ -1,8 +1,4 @@ -{"stream": "deliverystats", "data": {"InactiveMails": 0, "Bounces": [{"Name": "All", "Count": 0}]}, "emitted_at": 1667895141188} -{"stream": "message-streams", "data": {"MessageStreams": [{"ID": "broadcast", "ServerID": 9708911, "Name": "Default Broadcast Stream", "Description": "Default stream used for sending broadcast messages", "MessageStreamType": "Broadcasts", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "Postmark"}}, {"ID": "inbound", "ServerID": 9708911, "Name": "Default Inbound Stream", "Description": "Default stream used for receiving inbound messages", "MessageStreamType": "Inbound", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "None"}}, {"ID": "outbound", "ServerID": 9708911, "Name": "Default Transactional Stream", "Description": "Default stream used for sending transactional messages", "MessageStreamType": "Transactional", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "None"}}], "TotalCount": 3}, "emitted_at": 1667895141700} -{"stream": "servers", "data": {"TotalCount": 1, "Servers": []}, "emitted_at": 1667895142238} -{"stream": "messages", "data": {"TotalCount": 0, "Messages": []}, "emitted_at": 1667895142930} -{"stream": "domains", "data": {"TotalCount": 1, "Domains": []}, "emitted_at": 1667895143502} -{"stream": "bounces", "data": {"TotalCount": 0, "Bounces": []}, "emitted_at": 1667895144067} - - +{"stream": "deliverystats", "data": {"Name": "All", "Count": 0}, "emitted_at": 1668012739230} +{"stream": "message-streams", "data": {"ID": "broadcast", "ServerID": 9708911, "Name": "Default Broadcast Stream", "Description": "Default stream used for sending broadcast messages", "MessageStreamType": "Broadcasts", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "Postmark"}}, "emitted_at": 1668012739882} +{"stream": "message-streams", "data": {"ID": "inbound", "ServerID": 9708911, "Name": "Default Inbound Stream", "Description": "Default stream used for receiving inbound messages", "MessageStreamType": "Inbound", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "None"}}, "emitted_at": 1668012739885} +{"stream": "message-streams", "data": {"ID": "outbound", "ServerID": 9708911, "Name": "Default Transactional Stream", "Description": "Default stream used for sending transactional messages", "MessageStreamType": "Transactional", "CreatedAt": "2022-11-01T12:48:26-04:00", "UpdatedAt": null, "ArchivedAt": null, "ExpectedPurgeDate": null, "SubscriptionManagementConfiguration": {"UnsubscribeHandlingType": "None"}}, "emitted_at": 1668012739886} diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml index 4a93da15ca1cd..53ca6c7517eb4 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml @@ -4,6 +4,21 @@ definitions: selector: extractor: field_pointer: [] + selector_bounces: + extractor: + field_pointer: [ "Bounces" ] + selector_domains: + extractor: + field_pointer: [ "Domains" ] + selector_servers: + extractor: + field_pointer: [ "Servers" ] + selector_message_streams: + extractor: + field_pointer: [ "MessageStreams" ] + selector_messages: + extractor: + field_pointer: [ "Messages" ] requester: url_base: "https://api.postmarkapp.com" http_method: "GET" @@ -71,31 +86,51 @@ definitions: retriever: $ref: "*ref(definitions.retriever_account)" deliverystats: - $ref: "*ref(definitions.base_stream)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector_bounces)" $options: name: "deliverystats" - primary_key: "Type" + primary_key: "Name" path: "/deliverystats" message-streams: $ref: "*ref(definitions.base_stream)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector_message_streams)" $options: name: "message-streams" primary_key: "ID" path: "/message-streams" bounces: $ref: "*ref(definitions.base_stream)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector_bounces)" $options: name: "bounces" primary_key: "ID" path: "/bounces" + servers: $ref: "*ref(definitions.base_stream_account)" + retriever: + $ref: "*ref(definitions.retriever_account)" + record_selector: + $ref: "*ref(definitions.selector_servers)" $options: name: "servers" primary_key: "ID" path: "/servers" messages: $ref: "*ref(definitions.base_stream)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector_messages)" $options: name: "messages" primary_key: "MessageID" @@ -106,6 +141,10 @@ definitions: name: "domains" primary_key: "ID" path: "/domains" + retriever: + $ref: "*ref(definitions.retriever_account)" + record_selector: + $ref: "*ref(definitions.selector_domains)" streams: - "*ref(definitions.deliverystats)" - "*ref(definitions.message-streams)" @@ -117,4 +156,4 @@ streams: check: stream_names: - - "servers" + - "message-streams" diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/bounces.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/bounces.json index 0b59f7ec823d5..9f38e4c8eee5f 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/bounces.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/bounces.json @@ -2,64 +2,56 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { - "TotalCount": { + "ID": { "type": "integer" }, - "Bounces": { - "type": "object", - "properties": { - "ID": { - "type": "integer" - }, - "Type": { - "type": "string" - }, - "TypeCode": { - "type": "integer" - }, - "Name": { - "type": "string" - }, - "Tag": { - "type": "string" - }, - "MessageID": { - "type": "string" - }, - "ServerID": { - "type": "integer" - }, - "MessageStream": { - "type": "string" - }, - "Description": { - "type": "string" - }, - "Details": { - "type": "string" - }, - "Email": { - "type": "string" - }, - "From": { - "type": "string" - }, - "BouncedAt": { - "type": "string" - }, - "DumpAvailable": { - "type": "boolean" - }, - "Inactive": { - "type": "boolean" - }, - "CanActivate": { - "type": "boolean" - }, - "Subject": { - "type": "string" - } - } + "Type": { + "type": "string" + }, + "TypeCode": { + "type": "integer" + }, + "Name": { + "type": "string" + }, + "Tag": { + "type": "string" + }, + "MessageID": { + "type": "string" + }, + "ServerID": { + "type": "integer" + }, + "MessageStream": { + "type": "string" + }, + "Description": { + "type": "string" + }, + "Details": { + "type": "string" + }, + "Email": { + "type": "string" + }, + "From": { + "type": "string" + }, + "BouncedAt": { + "type": "string" + }, + "DumpAvailable": { + "type": "boolean" + }, + "Inactive": { + "type": "boolean" + }, + "CanActivate": { + "type": "boolean" + }, + "Subject": { + "type": "string" } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/deliverystats.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/deliverystats.json index ff3d6fffbcb54..9e90a936f52d2 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/deliverystats.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/deliverystats.json @@ -2,23 +2,11 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { - "InactiveMails": { - "type": "integer" + "Name": { + "type": "string" }, - "Bounces": { - "type": "array", - "items": - { - "type": "object", - "properties": { - "Name": { - "type": "string" - }, - "Count": { - "type": "integer" - } - } - } + "Count": { + "type": "integer" } } } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/domains.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/domains.json index 3cf12c51da9e7..8a956f39635a3 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/domains.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/domains.json @@ -2,36 +2,23 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { - "TotalCount": { - "type": "integer" + "Name": { + "type": "string" + }, + "SPFVerified": { + "type": "boolean" + }, + "DKIMVerified": { + "type": "boolean" }, - "Domains": { - "type": "array", - "items": - { - "type": "object", - "properties": { - "Name": { - "type": "string" - }, - "SPFVerified": { - "type": "boolean" - }, - "DKIMVerified": { - "type": "boolean" - }, - "WeakDKIM": { - "type": "boolean" - }, - "ReturnPathDomainVerified": { - "type": "boolean" - }, - "ID": { - "type": "integer" - } - } - } - + "WeakDKIM": { + "type": "boolean" + }, + "ReturnPathDomainVerified": { + "type": "boolean" + }, + "ID": { + "type": "integer" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json index baa31350a029a..5d3ac676065c5 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json @@ -2,12 +2,6 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", "properties": { - "TotalCount": { - "type": "integer" - }, - "Servers": { - "type": "array", - "items": { "ID": { "type": "integer" }, @@ -78,7 +72,5 @@ "EnableSmtpApiErrorHooks": { "type": "boolean" } - } - } } } \ No newline at end of file From 29ef88b6efff75a202268844a0e0844ab3f4796c Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Wed, 9 Nov 2022 17:09:01 -0300 Subject: [PATCH 06/10] correct source def --- .../init/src/main/resources/seed/source_definitions.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 254a41cb4a7d4..dcc9b0b1ac349 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1075,12 +1075,11 @@ icon: postgresql.svg sourceType: database releaseStage: generally_available -- name: Postmarkapp +- name: Postmark App sourceDefinitionId: cde75ca1-1e28-4a0f-85bb-90c546de9f1f dockerRepository: airbyte/source-postmarkapp - dockerImageTag: 0.0.1 + dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.com/integrations/sources/postmarkapp - icon: postmarkapp.svg sourceType: api releaseStage: alpha - name: Prestashop From ec13068f3d476d64a6b3088ba133f92bbfa52479 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Wed, 9 Nov 2022 17:14:03 -0300 Subject: [PATCH 07/10] correct build.md --- airbyte-integrations/builds.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 0cc7507f30190..0ac51d1c82125 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -174,4 +174,4 @@ | S3 | [![destination-s3](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-s3%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-s3) | | Scylla | [![destination-scylla](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-s3%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-scylla) | | SFTP-JSON | [![destination-sftp-json](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-sftp-json%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-sftp-json) | -| Snowflake | [![destination-snowflake](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-snowflake%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-snowflake) |\ +| Snowflake | [![destination-snowflake](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-snowflake%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-snowflake) |\ \ No newline at end of file From 5652c0b6e22aff4cdf6add163ce4cbd4b9db8b5d Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Wed, 9 Nov 2022 17:26:43 -0300 Subject: [PATCH 08/10] run format --- .../source_delighted/source.py | 2 +- .../integration_tests/configured_catalog.json | 72 ++++++++----------- .../source_postmarkapp/postmarkapp.yaml | 19 +++-- .../source_postmarkapp/schemas/bounces.json | 2 +- .../schemas/deliverystats.json | 2 +- .../schemas/message-streams.json | 2 +- .../source_postmarkapp/schemas/messages.json | 30 ++++---- .../schemas/outbound-stats.json | 2 +- .../source_postmarkapp/schemas/server.json | 9 ++- .../source_postmarkapp/schemas/servers.json | 9 ++- .../source_postmarkapp/schemas/templates.json | 2 +- .../source_postmarkapp/spec.yaml | 8 +-- 12 files changed, 71 insertions(+), 88 deletions(-) diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py index e4f2583db75dc..f584d5c987920 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/source.py +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/source.py @@ -118,7 +118,7 @@ class Bounces(IncrementalDelightedStream): primary_key = "person_id" def path(self, **kwargs) -> str: - return "deliverystats.json" + return "bounces.json" class SurveyResponses(IncrementalDelightedStream): diff --git a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json index deec84d6256b7..792ca62f03aab 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-postmarkapp/integration_tests/configured_catalog.json @@ -11,64 +11,54 @@ "destination_sync_mode": "overwrite" }, { - "stream" : { - "name" : "message-streams", - "json_schema" : {}, + "stream": { + "name": "message-streams", + "json_schema": {}, "source_defined_primary_key": [["ID"]], - "supported_sync_modes" : [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode" : "full_refresh", - "destination_sync_mode" : "overwrite" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { - "stream" : { - "name" : "servers", - "json_schema" : {}, + "stream": { + "name": "servers", + "json_schema": {}, "source_defined_primary_key": [["ID"]], - "supported_sync_modes" : [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode" : "full_refresh", - "destination_sync_mode" : "overwrite" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { - "stream" : { - "name" : "messages", - "json_schema" : {}, + "stream": { + "name": "messages", + "json_schema": {}, "source_defined_primary_key": [["MessageID"]], - "supported_sync_modes" : [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode" : "full_refresh", - "destination_sync_mode" : "overwrite" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { - "stream" : { - "name" : "domains", - "json_schema" : {}, + "stream": { + "name": "domains", + "json_schema": {}, "source_defined_primary_key": [["ID"]], - "supported_sync_modes" : [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode" : "full_refresh", - "destination_sync_mode" : "overwrite" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, - { - "stream" : { - "name" : "bounces", - "json_schema" : {}, + { + "stream": { + "name": "bounces", + "json_schema": {}, "source_defined_primary_key": [["ID"]], - "supported_sync_modes" : [ - "full_refresh" - ] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode" : "full_refresh", - "destination_sync_mode" : "overwrite" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml index 53ca6c7517eb4..ef5426aeb8558 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/postmarkapp.yaml @@ -6,27 +6,27 @@ definitions: field_pointer: [] selector_bounces: extractor: - field_pointer: [ "Bounces" ] + field_pointer: ["Bounces"] selector_domains: extractor: - field_pointer: [ "Domains" ] + field_pointer: ["Domains"] selector_servers: extractor: - field_pointer: [ "Servers" ] + field_pointer: ["Servers"] selector_message_streams: extractor: - field_pointer: [ "MessageStreams" ] + field_pointer: ["MessageStreams"] selector_messages: extractor: - field_pointer: [ "Messages" ] + field_pointer: ["Messages"] requester: url_base: "https://api.postmarkapp.com" http_method: "GET" request_options_provider: request_body_json: - Accept: "application/json" + Accept: "application/json" request_parameters: - count: '500' + count: "500" authenticator: type: "ApiKeyAuthenticator" header: "X-Postmark-Server-Token" @@ -36,9 +36,9 @@ definitions: http_method: "GET" request_options_provider: request_body_json: - Accept: "application/json" + Accept: "application/json" request_parameters: - count: '500' + count: "500" authenticator: type: "ApiKeyAuthenticator" header: "X-Postmark-Account-Token" @@ -153,7 +153,6 @@ streams: - "*ref(definitions.bounces)" - "*ref(definitions.servers)" - check: stream_names: - "message-streams" diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/bounces.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/bounces.json index 9f38e4c8eee5f..5e072c87c2ed2 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/bounces.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/bounces.json @@ -54,4 +54,4 @@ "type": "string" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/deliverystats.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/deliverystats.json index 9e90a936f52d2..627bfbc67816b 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/deliverystats.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/deliverystats.json @@ -9,4 +9,4 @@ "type": "integer" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/message-streams.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/message-streams.json index c79c10bc85113..48cff6d9a4999 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/message-streams.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/message-streams.json @@ -38,4 +38,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/messages.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/messages.json index 98807322b1556..5583a378f950f 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/messages.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/messages.json @@ -13,19 +13,17 @@ }, "To": { "type": "array", - "items": - { - "type": "object", - "properties": { - "Email": { - "type": "string" - }, - "Name": { - "type": "null" - } + "items": { + "type": "object", + "properties": { + "Email": { + "type": "string" + }, + "Name": { + "type": "null" } } - + } }, "Cc": { "type": "array", @@ -37,11 +35,9 @@ }, "Recipients": { "type": "array", - "items": - { - "type": "string" - } - + "items": { + "type": "string" + } }, "ReceivedAt": { "type": "string" @@ -80,4 +76,4 @@ "type": "boolean" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/outbound-stats.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/outbound-stats.json index b05e387519339..2d597f73e2782 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/outbound-stats.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/outbound-stats.json @@ -54,4 +54,4 @@ "type": "integer" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/server.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/server.json index 5d3ac676065c5..5e9cd760674a7 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/server.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/server.json @@ -10,10 +10,9 @@ }, "ApiTokens": { "type": "array", - "items": - { - "type": "string" - } + "items": { + "type": "string" + } }, "Color": { "type": "string" @@ -73,4 +72,4 @@ "type": "boolean" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json index 5d3ac676065c5..5e9cd760674a7 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/servers.json @@ -10,10 +10,9 @@ }, "ApiTokens": { "type": "array", - "items": - { - "type": "string" - } + "items": { + "type": "string" + } }, "Color": { "type": "string" @@ -73,4 +72,4 @@ "type": "boolean" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/templates.json b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/templates.json index fb5a8f3809f58..ce2e437b1c0da 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/templates.json +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/schemas/templates.json @@ -29,4 +29,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/spec.yaml b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/spec.yaml index d55d17e66572e..969662f1e1442 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/spec.yaml +++ b/airbyte-integrations/connectors/source-postmarkapp/source_postmarkapp/spec.yaml @@ -14,7 +14,7 @@ connectionSpecification: description: API Key for server airbyte_secret: true X-Postmark-Account-Token: - title: X-Postmark-Account-Token - type: string - description: API Key for account - airbyte_secret: true + title: X-Postmark-Account-Token + type: string + description: API Key for account + airbyte_secret: true From 68880762e26a8e3c5e47fd0023c391e76c81af8a Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Wed, 9 Nov 2022 17:29:31 -0300 Subject: [PATCH 09/10] update doc --- docs/integrations/sources/postmarkapp.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/integrations/sources/postmarkapp.md b/docs/integrations/sources/postmarkapp.md index 0cb1567433abf..76e23fbf4a856 100644 --- a/docs/integrations/sources/postmarkapp.md +++ b/docs/integrations/sources/postmarkapp.md @@ -16,7 +16,6 @@ Server-API - [Message-Streams](https://postmarkapp.com/developer/api/message-streams-api#list-message-streams): Lets you manage message streams for a specific server. Please note: A Server may have up to 10 Streams, including the default ones. Default Streams cannot be deleted, and Servers can only have 1 Inbound Stream. - [Outbound stats](https://account.postmarkapp.com/servers/9708911/credentials): Lets you get all of the statistics of your outbound emails for a specific server. These statistics are stored permantently and do not expire. All stats use EST timezone -TODO: Account-API - [Servers](https://postmarkapp.com/developer/api/servers-api): Lets you manage servers for a specific account. - [Domains](https://postmarkapp.com/developer/api/domains-api): Gets a list of domains containing an overview of the domain and authentication status. @@ -57,4 +56,4 @@ The Postmarkapp source connector supports the following [sync modes](https://doc | Version | Date | Pull Request | Subject | | :------ |:-----|:-------------| :----------------------------------------- | -| 0.1.0 | TBA | TBA | 🎉 New Source: Postmarkapp API [low-code CDK] | \ No newline at end of file +| 0.1.0 | 2022-11-09 | 18220 | 🎉 New Source: Postmarkapp API [low-code CDK] | \ No newline at end of file From 83060a7c15fd20c3ff605b02cc4b90e077887f1d Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 9 Nov 2022 20:50:27 +0000 Subject: [PATCH 10/10] auto-bump connector version --- .../src/main/resources/seed/source_specs.yaml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 51d5a7584a76e..c729b4c6949be 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -10557,6 +10557,31 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-postmarkapp:0.1.0" + spec: + documentationUrl: "https://docsurl.com" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Postmarkapp Spec" + type: "object" + required: + - "X-Postmark-Server-Token" + - "X-Postmark-Account-Token" + additionalProperties: true + properties: + X-Postmark-Server-Token: + title: "X-Postmark-Server-Token" + type: "string" + description: "API Key for server" + airbyte_secret: true + X-Postmark-Account-Token: + title: "X-Postmark-Account-Token" + type: "string" + description: "API Key for account" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-prestashop:0.2.0" spec: documentationUrl: "https://docsurl.com"