From 92aeac3e2b769d06dbba390ac61a800ce5662c9d Mon Sep 17 00:00:00 2001 From: David Thompson Date: Tue, 16 Aug 2022 19:58:27 +0100 Subject: [PATCH] :tada: New Source: AppFollow Service (#14418) * Generated template Using https://docs.airbyte.com/connector-development/tutorials/cdk-speedrun * Added spec.yaml Ref https://appfollow.docs.apiary.io/ Does not currently include authentication * Schema and working discover for ratings API * Add catalog * Add cid as required parameter * Working ratings call BASIC AUTH REALLY * Use basic auth in check connection * Updated comment for request params * Update README for local run * Moved with common params to stream superclass * Add support for country parameter * WIP: acceptance 90% pass * Passing acceptance tests for ratings in docker * Remove incremental unit tests We have no incremental streams * Match number of streams to test * Passing streams unit tests * Connection check test * Update unit test type check * Remove TODO * fix: tests are failing * docs: added documentation for appfollow and logo * docs: added documentation for appfollow and logo * Bump io.airbyte.version * fix: tests are failing * auto-bump connector version [ci skip] Co-authored-by: Harshith Mullapudi Co-authored-by: Octavia Squidington III --- .../src/main/resources/icons/appfollow.svg | 18 +++ .../resources/seed/source_definitions.yaml | 8 ++ .../src/main/resources/seed/source_specs.yaml | 35 +++++ .../connectors/source-appfollow/.dockerignore | 6 + .../connectors/source-appfollow/Dockerfile | 38 +++++ .../connectors/source-appfollow/README.md | 132 ++++++++++++++++++ .../acceptance-test-config.yml | 22 +++ .../acceptance-test-docker.sh | 16 +++ .../connectors/source-appfollow/build.gradle | 9 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/acceptance.py | 14 ++ .../integration_tests/configured_catalog.json | 29 ++++ .../integration_tests/invalid_config.json | 6 + .../integration_tests/sample_config.json | 6 + .../connectors/source-appfollow/main.py | 13 ++ .../source-appfollow/requirements.txt | 2 + .../source-appfollow/sample_files/config.json | 7 + .../sample_files/configured_catalog.json | 27 ++++ .../sample_files/invalid_config.json | 3 + .../connectors/source-appfollow/setup.py | 25 ++++ .../source_appfollow/__init__.py | 8 ++ .../source_appfollow/schemas/ratings.json | 64 +++++++++ .../source_appfollow/source.py | 108 ++++++++++++++ .../source_appfollow/spec.yaml | 29 ++++ .../source-appfollow/unit_tests/__init__.py | 3 + .../unit_tests/test_source.py | 29 ++++ .../unit_tests/test_streams.py | 73 ++++++++++ docs/integrations/sources/appfollow.md | 40 ++++++ 28 files changed, 773 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/icons/appfollow.svg create mode 100644 airbyte-integrations/connectors/source-appfollow/.dockerignore create mode 100644 airbyte-integrations/connectors/source-appfollow/Dockerfile create mode 100644 airbyte-integrations/connectors/source-appfollow/README.md create mode 100644 airbyte-integrations/connectors/source-appfollow/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-appfollow/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-appfollow/build.gradle create mode 100644 airbyte-integrations/connectors/source-appfollow/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-appfollow/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-appfollow/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-appfollow/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-appfollow/main.py create mode 100644 airbyte-integrations/connectors/source-appfollow/requirements.txt create mode 100644 airbyte-integrations/connectors/source-appfollow/sample_files/config.json create mode 100644 airbyte-integrations/connectors/source-appfollow/sample_files/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-appfollow/sample_files/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-appfollow/setup.py create mode 100644 airbyte-integrations/connectors/source-appfollow/source_appfollow/__init__.py create mode 100644 airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/ratings.json create mode 100644 airbyte-integrations/connectors/source-appfollow/source_appfollow/source.py create mode 100644 airbyte-integrations/connectors/source-appfollow/source_appfollow/spec.yaml create mode 100644 airbyte-integrations/connectors/source-appfollow/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-appfollow/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-appfollow/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/appfollow.md diff --git a/airbyte-config/init/src/main/resources/icons/appfollow.svg b/airbyte-config/init/src/main/resources/icons/appfollow.svg new file mode 100644 index 0000000000000..a182781eec30e --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/appfollow.svg @@ -0,0 +1,18 @@ + + + + + + + + + diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index b870dc0e4b789..daecfed3058a5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -53,6 +53,14 @@ icon: apify.svg sourceType: api releaseStage: alpha +- name: Appfollow + sourceDefinitionId: b4375641-e270-41d3-9c20-4f9cecad87a8 + dockerRepository: airbyte/source-appfollow + dockerImageTag: 0.1.1 + documentationUrl: https://docs.airbyte.io/integrations/sources/appfollow + icon: appfollow.svg + sourceType: api + releaseStage: alpha - name: Appstore sourceDefinitionId: 2af123bf-0aaf-4e0d-9784-cb497f23741a dockerRepository: airbyte/source-appstore-singer diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 52660d3de78e2..bdba040ba4d1f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -605,6 +605,41 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-appfollow:0.1.1" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/appfollow" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Appfollow Spec" + type: "object" + required: + - "ext_id" + - "country" + - "cid" + - "api_secret" + additionalProperties: true + properties: + ext_id: + type: "string" + title: "app external id" + description: "for App Store — this is 9-10 digits identification number;\ + \ for Google Play — this is bundle name;" + cid: + type: "string" + title: "client id" + description: "client id provided by Appfollow" + api_secret: + type: "string" + title: "api secret" + description: "api secret provided by Appfollow" + airbyte_secret: true + country: + type: "string" + title: "country" + description: "getting data by Country" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-appstore-singer:0.2.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/appstore" diff --git a/airbyte-integrations/connectors/source-appfollow/.dockerignore b/airbyte-integrations/connectors/source-appfollow/.dockerignore new file mode 100644 index 0000000000000..c4606cadee2df --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_appfollow +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-appfollow/Dockerfile b/airbyte-integrations/connectors/source-appfollow/Dockerfile new file mode 100644 index 0000000000000..8b333c731db82 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_appfollow ./source_appfollow + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.name=airbyte/source-appfollow diff --git a/airbyte-integrations/connectors/source-appfollow/README.md b/airbyte-integrations/connectors/source-appfollow/README.md new file mode 100644 index 0000000000000..0dc128693ac43 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/README.md @@ -0,0 +1,132 @@ +# Appfollow Source + +This is the repository for the Appfollow source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/appfollow). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-appfollow:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/appfollow) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_appfollow/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source appfollow test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python3 main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-appfollow:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-appfollow:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-appfollow:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-appfollow:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-appfollow:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-appfollow:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-appfollow:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-appfollow:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-appfollow/acceptance-test-config.yml b/airbyte-integrations/connectors/source-appfollow/acceptance-test-config.yml new file mode 100644 index 0000000000000..3d14fbd403838 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/acceptance-test-config.yml @@ -0,0 +1,22 @@ +# # See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# # for more information about how to configure these tests + +connector_image: airbyte/source-appfollow:dev +tests: + spec: + - spec_path: "source_appfollow/spec.yaml" +# Enable these once we get appfollow credentials +# connection: +# - config_path: "secrets/config.json" +# status: "succeed" +# - config_path: "integration_tests/invalid_config.json" +# status: "failed" +# discovery: +# - config_path: "secrets/config.json" +# basic_read: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# empty_streams: [] +# full_refresh: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-appfollow/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-appfollow/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-appfollow/build.gradle b/airbyte-integrations/connectors/source-appfollow/build.gradle new file mode 100644 index 0000000000000..d97a44c6cb5cd --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_appfollow' +} diff --git a/airbyte-integrations/connectors/source-appfollow/integration_tests/__init__.py b/airbyte-integrations/connectors/source-appfollow/integration_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py new file mode 100644 index 0000000000000..950b53b59d416 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-appfollow/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-appfollow/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..966eb90eb510f --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/integration_tests/configured_catalog.json @@ -0,0 +1,29 @@ +{ + "streams": [ + { + "stream": { + "name": "ratings", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "ext_id": { + "type": "string" + }, + "cid": { + "type": "string" + }, + "country": { + "type": "string" + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-appfollow/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-appfollow/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..f2311f73ce286 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "ext_id": "FIXME", + "cid": "FIXME", + "api_secret": "FIXME", + "country": "FIXME" +} diff --git a/airbyte-integrations/connectors/source-appfollow/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-appfollow/integration_tests/sample_config.json new file mode 100644 index 0000000000000..f2311f73ce286 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "ext_id": "FIXME", + "cid": "FIXME", + "api_secret": "FIXME", + "country": "FIXME" +} diff --git a/airbyte-integrations/connectors/source-appfollow/main.py b/airbyte-integrations/connectors/source-appfollow/main.py new file mode 100644 index 0000000000000..b015bf8942c8f --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_appfollow import SourceAppfollow + +if __name__ == "__main__": + source = SourceAppfollow() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-appfollow/requirements.txt b/airbyte-integrations/connectors/source-appfollow/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-appfollow/sample_files/config.json b/airbyte-integrations/connectors/source-appfollow/sample_files/config.json new file mode 100644 index 0000000000000..c2e9f9c5c6d16 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/sample_files/config.json @@ -0,0 +1,7 @@ +{ + "ext_id": "FIXME", + "cid": "FIXME", + "api_secret": "FIXME", + "country": "FIXME" +} + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-appfollow/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-appfollow/sample_files/configured_catalog.json new file mode 100644 index 0000000000000..2720a2e328a0a --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/sample_files/configured_catalog.json @@ -0,0 +1,27 @@ +{ + "streams": [ + { + "stream": { + "name": "ratings", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "ext_id": { + "type": "string" + }, + "cid": { + "type": "string" + }, + "country": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-appfollow/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-appfollow/sample_files/invalid_config.json new file mode 100644 index 0000000000000..7905daf0a25bd --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/sample_files/invalid_config.json @@ -0,0 +1,3 @@ +{ + "ext_id": "bar" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-appfollow/setup.py b/airbyte-integrations/connectors/source-appfollow/setup.py new file mode 100644 index 0000000000000..2e4a21ee3d081 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/setup.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1.56", +] + +TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6.1", "source-acceptance-test", "requests_mock~=1.9"] + +setup( + name="source_appfollow", + description="Source implementation for Appfollow.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/__init__.py b/airbyte-integrations/connectors/source-appfollow/source_appfollow/__init__.py new file mode 100644 index 0000000000000..4b57b83ba3808 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceAppfollow + +__all__ = ["SourceAppfollow"] diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/ratings.json b/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/ratings.json new file mode 100644 index 0000000000000..601a5c81d62fb --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/schemas/ratings.json @@ -0,0 +1,64 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ratings": { + "list": { + "type": [ + "null", + "array" + ], + "items": { + "properties": { + "stars2": { + "type": "integer" + }, + "version": { + "type": "string" + }, + "stars1": { + "type": "integer" + }, + "stars_total": { + "type": "integer" + }, + "country": { + "type": "string" + }, + "date": { + "type": "string" + }, + "stars4": { + "type": "integer" + }, + "stars5": { + "type": "integer" + }, + "stars3": { + "type": "integer" + }, + "rating": { + "type": "number" + } + }, + "type": [ + "null", + "object" + ] + } + }, + "ext_id": { + "type": [ + "null", + "string" + ] + }, + "store": { + "type": [ + "null", + "string" + ] + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/source.py b/airbyte-integrations/connectors/source-appfollow/source_appfollow/source.py new file mode 100644 index 0000000000000..80a3a50ec2cb1 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/source.py @@ -0,0 +1,108 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator +from requests.auth import HTTPBasicAuth + +logger = logging.getLogger("airbyte") + +# Basic full refresh stream + + +class AppfollowStream(HttpStream, ABC): + + url_base = "https://api.appfollow.io/" + + def __init__(self, ext_id: str, cid: str, **kwargs): + super().__init__(**kwargs) + self.ext_id = ext_id + self.cid = cid + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + Include common app and client parameters + """ + return {"ext_id": self.ext_id, "cid": self.cid} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + :return an iterable containing each record in the response + """ + response_json = response.json() + yield response_json + + +class Ratings(AppfollowStream): + """ + Ratings is a stream that pulls app ratings data from the Appfollow API. + """ + + primary_key = None + + def __init__(self, country: str, **kwargs): + super().__init__(**kwargs) + self.country = country + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state, stream_slice, next_page_token) + params["country"] = self.country + return params + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "ratings" + + +# Source + + +class SourceAppfollow(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + A connection check to validate that the user-provided config can be used to connect to the underlying API + + :param config: the user-input config object conforming to the connector's spec.yaml + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + logger.info("Checking Appfollow API connection...") + try: + ext_id = config["ext_id"] + cid = config["cid"] + api_secret = config["api_secret"] + response = requests.get( + f"https://api.appfollow.io/ratings?ext_id={ext_id}&cid={cid}", auth=HTTPBasicAuth(api_secret, api_secret) + ) + if response.status_code == 200: + return True, None + else: + return False, "Invalid Appfollow API credentials" + except Exception as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + Appfollow streams + + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + auth = BasicHttpAuthenticator(username=config["api_secret"], password=config["api_secret"]) + args = {"ext_id": config["ext_id"], "cid": config["cid"]} + return [Ratings(authenticator=auth, country=config["country"], **args)] diff --git a/airbyte-integrations/connectors/source-appfollow/source_appfollow/spec.yaml b/airbyte-integrations/connectors/source-appfollow/source_appfollow/spec.yaml new file mode 100644 index 0000000000000..ef66540c0877c --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/source_appfollow/spec.yaml @@ -0,0 +1,29 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/appfollow +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Appfollow Spec + type: object + required: + - ext_id + - country + - cid + - api_secret + additionalProperties: true + properties: + ext_id: + type: string + title: app external id + description: for App Store — this is 9-10 digits identification number; for Google Play — this is bundle name; + cid: + type: string + title: client id + description: client id provided by Appfollow + api_secret: + type: string + title: api secret + description: api secret provided by Appfollow + airbyte_secret: true + country: + type: string + title: country + description: getting data by Country diff --git a/airbyte-integrations/connectors/source-appfollow/unit_tests/__init__.py b/airbyte-integrations/connectors/source-appfollow/unit_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-appfollow/unit_tests/test_source.py b/airbyte-integrations/connectors/source-appfollow/unit_tests/test_source.py new file mode 100644 index 0000000000000..0310c1b282277 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/unit_tests/test_source.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_appfollow.source import SourceAppfollow + + +def test_check_connection(mocker, requests_mock): + source = SourceAppfollow() + logger_mock, config_mock = MagicMock(), MagicMock() + + # success + requests_mock.get("https://api.appfollow.io/ratings", json={"data": "pong!"}) + assert source.check_connection(logger_mock, config_mock) == (True, None) + + # failure + requests_mock.get("https://api.appfollow.io/ratings", status_code=500) + ok, err = source.check_connection(logger_mock, config_mock) + assert (ok, type(err)) == (False, str) + + +def test_streams(mocker): + source = SourceAppfollow() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 1 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-appfollow/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-appfollow/unit_tests/test_streams.py new file mode 100644 index 0000000000000..98d81c9ec9377 --- /dev/null +++ b/airbyte-integrations/connectors/source-appfollow/unit_tests/test_streams.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_appfollow.source import AppfollowStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + def __init__(self): + self.ext_id = "00000" + self.cid = "000000" + + mocker.patch.object(AppfollowStream, "__init__", __init__) + mocker.patch.object(AppfollowStream, "path", "v0/example_endpoint") + mocker.patch.object(AppfollowStream, "primary_key", "test_primary_key") + mocker.patch.object(AppfollowStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = AppfollowStream() + inputs = {"stream_state": "test_stream_state"} + expected_params = {"ext_id": "00000", "cid": "000000"} + assert stream.request_params(**inputs) == expected_params + + +def test_parse_response(patch_base_class): + stream = AppfollowStream() + mock_response = MagicMock() + inputs = {"stream_state": "test_stream_state", "response": mock_response} + expected_parsed_object = mock_response.json() + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = AppfollowStream() + inputs = {"stream_state": "test_stream_state"} + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = AppfollowStream() + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = AppfollowStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = AppfollowStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/sources/appfollow.md b/docs/integrations/sources/appfollow.md new file mode 100644 index 0000000000000..95d57f4badd7c --- /dev/null +++ b/docs/integrations/sources/appfollow.md @@ -0,0 +1,40 @@ +# Appfollow + +This page guides you through setting up the Appfollow source connector to sync data for the [Appfollow API](https://appfollow.docs.apiary.io/#introduction/api-methods). + +## Prerequisite + +To set up the Appfollow source connector, you'll need your Appfollow `ext_id`, `cid`, `api_secret` and `Country`. + +## Set up the Appfollow source connector + +1. Log into your [Airbyte Cloud](https://cloud.airbyte.io/workspaces) or Airbyte OSS account. +2. Click **Sources** and then click **+ New source**. +3. On the Set up the source page, select **Appfollow** from the Source type dropdown. +4. Enter a name for your source. +5. For **ext_id**, **cid**, **api_secret** and **Country**, enter the Appfollow ext_id, cid, api_secret and country. +6. Click **Set up source**. + +## Supported Streams + +The Appfollow source connector supports the following streams: + +- [Ratings](https://appfollow.docs.apiary.io/#reference/0/9.-ratings) \(Full Refresh sync\) + +If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) + +## Supported sync modes + +The Appfollow source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +- Full Refresh + +## Performance considerations + +The Appfollow connector ideally should gracefully handle Appfollow API limitations under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------- | +| 0.1.0 | 2022-08-11 | [14418](https://github.com/airbytehq/airbyte/pull/14418) | New Source: Appfollow |