From 563870cb498dd126207106f4dea4c91fa3e75aaf Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Wed, 10 Aug 2022 16:15:01 -0400 Subject: [PATCH] :tada: New Source: Hubplanner (#15521) * added hubplanner source connector * feat: added more streams to read from * cleaned up some unneeded integration tests * fix hubplanner schema * changes * update dockerfile * add seed and doc * update spec * run source spec seed file Co-authored-by: Ricky Renner --- .../resources/seed/source_definitions.yaml | 7 + .../src/main/resources/seed/source_specs.yaml | 19 + .../source-hubplanner/.dockerignore | 6 + .../connectors/source-hubplanner/Dockerfile | 16 + .../connectors/source-hubplanner/README.md | 132 +++++ .../acceptance-test-config.yml | 26 + .../acceptance-test-docker.sh | 16 + .../connectors/source-hubplanner/build.gradle | 9 + .../integration_tests/__init__.py | 3 + .../integration_tests/acceptance.py | 16 + .../integration_tests/configured_catalog.json | 67 +++ .../integration_tests/invalid_config.json | 3 + .../connectors/source-hubplanner/main.py | 13 + .../source-hubplanner/requirements.txt | 2 + .../sample_files/configured_catalog.json | 137 +++++ .../connectors/source-hubplanner/setup.py | 29 ++ .../source_hubplanner/__init__.py | 8 + .../schemas/billing_rates.json | 45 ++ .../source_hubplanner/schemas/bookings.json | 264 ++++++++++ .../source_hubplanner/schemas/clients.json | 48 ++ .../source_hubplanner/schemas/events.json | 45 ++ .../source_hubplanner/schemas/holidays.json | 43 ++ .../source_hubplanner/schemas/projects.json | 490 ++++++++++++++++++ .../source_hubplanner/schemas/resources.json | 212 ++++++++ .../source_hubplanner/source.py | 248 +++++++++ .../source_hubplanner/spec.json | 17 + .../source-hubplanner/unit_tests/__init__.py | 3 + .../unit_tests/test_source.py | 15 + .../unit_tests/test_streams.py | 79 +++ docs/integrations/sources/hubplanner.md | 42 ++ 30 files changed, 2060 insertions(+) create mode 100644 airbyte-integrations/connectors/source-hubplanner/.dockerignore create mode 100644 airbyte-integrations/connectors/source-hubplanner/Dockerfile create mode 100644 airbyte-integrations/connectors/source-hubplanner/README.md create mode 100644 airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-hubplanner/build.gradle create mode 100644 airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-hubplanner/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-hubplanner/main.py create mode 100644 airbyte-integrations/connectors/source-hubplanner/requirements.txt create mode 100644 airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-hubplanner/setup.py create mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py create mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/billing_rates.json create mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json create mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/clients.json create mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/events.json create mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/holidays.json create mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json create mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json create mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py create mode 100644 airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json create mode 100644 airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/hubplanner.md diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 51501cd02aa1e..785db71a42414 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -411,6 +411,13 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/hellobaton sourceType: api releaseStage: alpha +- name: Hubplanner + sourceDefinitionId: 8097ceb9-383f-42f6-9f92-d3fd4bcc7689 + dockerRepository: airbyte/source-hubplanner + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/hubplanner + sourceType: api + releaseStage: alpha - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index d5bf94d8c27d3..079dad4ead84c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3703,6 +3703,25 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-hubplanner:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/hubplanner" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Hubplanner Spec" + type: "object" + required: + - "api_key" + additionalProperties: true + properties: + api_key: + type: "string" + description: "Hubplanner API key. See https://github.com/hubplanner/API#authentication\ + \ for more details." + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-hubspot:0.1.81" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" diff --git a/airbyte-integrations/connectors/source-hubplanner/.dockerignore b/airbyte-integrations/connectors/source-hubplanner/.dockerignore new file mode 100644 index 0000000000000..beb33d510fd42 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_hubplanner +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-hubplanner/Dockerfile b/airbyte-integrations/connectors/source-hubplanner/Dockerfile new file mode 100644 index 0000000000000..ba50ba0758fba --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.9-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_hubplanner ./source_hubplanner +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-hubplanner diff --git a/airbyte-integrations/connectors/source-hubplanner/README.md b/airbyte-integrations/connectors/source-hubplanner/README.md new file mode 100644 index 0000000000000..f38554dc57851 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/README.md @@ -0,0 +1,132 @@ +# Hubplanner Source + +This is the repository for the Hubplanner source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/hubplanner). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-hubplanner:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/hubplanner) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubplanner/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source hubplanner test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-hubplanner:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-hubplanner:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-hubplanner:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubplanner:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubplanner:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-hubplanner:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-hubplanner:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-hubplanner:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml new file mode 100644 index 0000000000000..7dece07aad48f --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml @@ -0,0 +1,26 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-hubplanner:dev +tests: + spec: + - spec_path: "source_hubplanner/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: ["holidays"] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-hubplanner/build.gradle b/airbyte-integrations/connectors/source-hubplanner/build.gradle new file mode 100644 index 0000000000000..c75aea3d3c219 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_hubplanner' +} diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py b/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py new file mode 100644 index 0000000000000..1302b2f57e10e --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-hubplanner/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..1404db601f4b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/configured_catalog.json @@ -0,0 +1,67 @@ +{ + "streams": [ + { + "stream": { + "name": "billing_rates", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "bookings", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "clients", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "holidays", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "projects", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "resources", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..92a71d900e591 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "invalid-api-key" +} diff --git a/airbyte-integrations/connectors/source-hubplanner/main.py b/airbyte-integrations/connectors/source-hubplanner/main.py new file mode 100644 index 0000000000000..14edc91deb050 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_hubplanner import SourceHubplanner + +if __name__ == "__main__": + source = SourceHubplanner() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-hubplanner/requirements.txt b/airbyte-integrations/connectors/source-hubplanner/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json new file mode 100644 index 0000000000000..6e00688736fa0 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json @@ -0,0 +1,137 @@ +{ + "streams": [ + { + "stream": { + "name": "billing_rates", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "bookings", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "clients", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "events", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "holidays", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "projects", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "resources", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See here for more details.", + "airbyte_secret": true + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubplanner/setup.py b/airbyte-integrations/connectors/source-hubplanner/setup.py new file mode 100644 index 0000000000000..cd9e6accd8555 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_hubplanner", + description="Source implementation for Hubplanner.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py new file mode 100644 index 0000000000000..3bcd0c2b2b7ae --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceHubplanner + +__all__ = ["SourceHubplanner"] diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/billing_rates.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/billing_rates.json new file mode 100644 index 0000000000000..810ab001a5bdb --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/billing_rates.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": [ + "null", + "string" + ] + }, + "rate": { + "type": [ + "null", + "number" + ] + }, + "metadata": { + "type": [ + "null", + "string" + ] + }, + "createdDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updatedDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "label": { + "type": "string" + }, + "currency": { + "type": "string" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json new file mode 100644 index 0000000000000..e40f7ed91bcc9 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json @@ -0,0 +1,264 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "allDay": { + "type": [ + "null", + "boolean" + ] + }, + "scale": { + "type": [ + "null", + "string" + ] + }, + "start": { + "type": "string" + }, + "end": { + "type": "string" + }, + "categoryTemplateId": { + "type": [ + "null", + "string" + ] + }, + "categoryName": { + "type": [ + "null", + "string" + ] + }, + "stateValue": { + "type": [ + "null", + "integer" + ] + }, + "resource": { + "type": "string" + }, + "project": { + "type": "string" + }, + "note": { + "type": [ + "null", + "string" + ] + }, + "details": { + "properties": { + "offDaysCount": { + "type": [ + "null", + "integer" + ] + }, + "workDaysCount": { + "type": [ + "null", + "integer" + ] + }, + "holidayCount": { + "type": [ + "null", + "integer" + ] + }, + "workWeekDetails": { + "items": { + "type": [ + "null", + "integer" + ] + }, + "type": [ + "null", + "array" + ] + }, + "bookedMinutes": { + "type": [ + "null", + "integer" + ] + }, + "budgetBookedAmount": { + "type": [ + "null", + "integer" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "createdDate": { + "format": "date-time", + "type": [ + "null", + "string" + ] + }, + "updatedDate": { + "format": "date-time", + "type": [ + "null", + "string" + ] + }, + "metadata": { + "type": [ + "null", + "string" + ] + }, + "customFields": { + "items": { + "properties": { + "_id": { + "type": "string" + }, + "templateId": { + "type": "string" + }, + "templateType": { + "type": "string" + }, + "templateLabel": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + }, + "choices": { + "items": { + "properties": { + "_id": { + "type": "string" + }, + "choiceId": { + "type": "string" + }, + "value": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + }, + "bookingRate": { + "properties": { + "external": { + "properties": { + "defaultRateId": { + "type": [ + "null", + "integer" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "internal": { + "properties": { + "defaultRateId": { + "type": [ + "null", + "integer" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "bookingCreatorId": { + "type": [ + "null", + "string" + ] + }, + "lastUpdatedById": { + "type": [ + "null", + "string" + ] + }, + "backgroundColor": { + "type": [ + "null", + "string" + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/clients.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/clients.json new file mode 100644 index 0000000000000..1b3e5ea8ab566 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/clients.json @@ -0,0 +1,48 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": "string" + }, + "company": { + "type": [ + "null", + "string" + ] + }, + "__v": { + "type": [ + "null", + "integer" + ] + }, + "createdDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updatedDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "deleted": { + "type": [ + "null", + "boolean" + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/events.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/events.json new file mode 100644 index 0000000000000..c06d633ef56c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/events.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": "string" + }, + "createdDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updatedDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "backgroundColor": { + "type": [ + "null", + "string" + ] + }, + "label": { + "type": "string" + }, + "metadata": { + "type": [ + "null", + "string" + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/holidays.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/holidays.json new file mode 100644 index 0000000000000..b210ed04dae6e --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/holidays.json @@ -0,0 +1,43 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": "string" + }, + "date": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "metadata": { + "type": [ + "null", + "string" + ] + }, + "createdDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updatedDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json new file mode 100644 index 0000000000000..f7f7c61eb8ce7 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json @@ -0,0 +1,490 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": "string" + }, + "links": { + "properties": {}, + "type": [ + "null", + "object" + ] + }, + "notes": { + "type": [ + "null", + "string" + ] + }, + "createdDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updatedDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "timeEntryEnabled": { + "type": [ + "null", + "boolean" + ] + }, + "timeEntryLocked": { + "type": [ + "null", + "boolean" + ] + }, + "timeEntryApproval": { + "type": [ + "null", + "boolean" + ] + }, + "resourceRates": { + "items": { + "properties": { + "resource": { + "type": [ + "null", + "string" + ] + }, + "companyBillingRateId": { + "type": [ + "null", + "string" + ] + }, + "companyBillingRate": { + "type": [ + "null", + "number" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + }, + "includeBookedTimeReports": { + "type": [ + "null", + "boolean" + ] + }, + "includeBookedTimeGrid": { + "type": [ + "null", + "boolean" + ] + }, + "projectManagers": { + "items": { + "type": [ + "null", + "string" + ] + }, + "type": [ + "null", + "array" + ] + }, + "resources": { + "items": { + "type": [ + "null", + "string" + ] + }, + "type": [ + "null", + "array" + ] + }, + "workDays": { + "items": { + "type": [ + "null", + "boolean" + ] + }, + "type": [ + "null", + "array" + ] + }, + "useProjectDays": { + "type": [ + "null", + "boolean" + ] + }, + "budget": { + "properties": { + "hasBudget": { + "type": [ + "null", + "boolean" + ] + }, + "projectHours": { + "properties": { + "active": { + "type": [ + "null", + "boolean" + ] + }, + "hours": { + "type": [ + "null", + "number" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "cashAmount": { + "properties": { + "active": { + "type": [ + "null", + "boolean" + ] + }, + "amount": { + "type": [ + "null", + "number" + ] + }, + "currency": { + "type": [ + "null", + "string" + ] + }, + "billingRate": { + "properties": { + "useDefault": { + "type": [ + "null", + "boolean" + ] + }, + "rate": { + "type": [ + "null", + "number" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "companyBillingRateId": { + "type": [ + "null", + "string" + ] + }, + "budgetHours": { + "type": [ + "null", + "number" + ] + }, + "budgetCashAmount": { + "type": [ + "null", + "number" + ] + }, + "budgetCurrency": { + "type": [ + "null", + "string" + ] + }, + "useStatusColor": { + "type": [ + "null", + "boolean" + ] + }, + "status": { + "type": [ + "null", + "string" + ] + }, + "useProjectDuration": { + "type": [ + "null", + "boolean" + ] + }, + "start": { + "type": [ + "null", + "string" + ] + }, + "end": { + "type": [ + "null", + "string" + ] + }, + "backgroundColor": { + "type": [ + "null", + "string" + ] + }, + "projectCode": { + "type": [ + "null", + "string" + ] + }, + "metadata": { + "type": [ + "null", + "string" + ] + }, + "customFields": { + "items": { + "properties": { + "_id": { + "type": "string" + }, + "templateId": { + "type": "string" + }, + "templateType": { + "type": "string" + }, + "templateLabel": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + }, + "choices": { + "items": { + "properties": { + "_id": { + "type": "string" + }, + "choiceId": { + "type": "string" + }, + "value": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + }, + "timeEntryNoteRequired": { + "type": [ + "null", + "boolean" + ] + }, + "projectRate": { + "properties": { + "external": { + "properties": { + "defaultRateId": { + "type": [ + "null", + "string" + ] + }, + "customRates": { + "items": { + "properties": { + "_id": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "resourceId": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "internal": { + "properties": { + "customRates": { + "items": { + "properties": { + "_id": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "resourceId": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + }, + "defaultRateId": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "note": { + "type": [ + "null", + "string" + ] + }, + "tags": { + "items": { + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json new file mode 100644 index 0000000000000..996f3435627a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json @@ -0,0 +1,212 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_id": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + }, + "metadata": { + "type": [ + "null", + "string" + ] + }, + "createdDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updatedDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "note": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": "string" + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "status": { + "type": [ + "null", + "string" + ] + }, + "role": { + "type": [ + "null", + "string" + ] + }, + "links": { + "properties": {}, + "type": [ + "null", + "object" + ] + }, + "billing": { + "properties": { + "useDefault": { + "type": [ + "null", + "boolean" + ] + }, + "rate": { + "type": [ + "null", + "number" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "useCustomAvailability": { + "type": [ + "null", + "boolean" + ] + }, + "customFields": { + "items": { + "properties": { + "_id": { + "type": "string" + }, + "templateId": { + "type": "string" + }, + "templateType": { + "type": "string" + }, + "templateLabel": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + }, + "choices": { + "items": { + "properties": { + "_id": { + "type": "string" + }, + "choiceId": { + "type": "string" + }, + "value": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + }, + "resourceRates": { + "properties": { + "external": { + "items": { + "properties": {}, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + }, + "internal": { + "items": { + "properties": {}, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "isProjectManager": { + "type": [ + "null", + "boolean" + ] + }, + "tags": { + "items": { + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py new file mode 100644 index 0000000000000..3ca0496d538bf --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py @@ -0,0 +1,248 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator + + +# Basic full refresh stream +class HubplannerStream(HttpStream, ABC): + + url_base = "https://api.hubplanner.com/v1" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. + + This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed + to most other methods in this class to help you form headers, request bodies, query params, etc.. + + For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a + 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. + The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. + If there are no more pages in the result, return None. + """ + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + return {} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + yield {} + + +# Basic incremental stream +class IncrementalHubplannerStream(HubplannerStream, ABC): + """ + TODO fill in details of this class to implement functionality related to incremental syncs for your connector. + if you do not need to implement incremental sync for any streams, remove this class. + """ + + # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. + state_checkpoint_interval = None + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return [] + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + return {} + + +class HubplannerAuthenticator(HttpAuthenticator): + def __init__(self, token: str, auth_header: str = "Authorization"): + self.auth_header = auth_header + self._token = token + + def get_auth_header(self) -> Mapping[str, Any]: + return {self.auth_header: f"{self._token}"} + + +class BillingRates(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/billingRate" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Bookings(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/booking" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Clients(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/client" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Events(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/event" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Holidays(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/holiday" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Projects(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/project" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +class Resources(HubplannerStream): + primary_key = "_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "v1/resource" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return super().request_params(stream_state, stream_slice, next_page_token) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + return response.json() + + +# Source +class SourceHubplanner(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + + url_base = "https://api.hubplanner.com/v1" + + try: + url = f"{url_base}/project" + + authenticator = HubplannerAuthenticator(token=config["api_key"]) + + session = requests.get(url, headers=authenticator.get_auth_header()) + session.raise_for_status() + + return True, None + except requests.exceptions.RequestException as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + authenticator = HubplannerAuthenticator(token=config["api_key"]) + return [ + BillingRates(authenticator=authenticator), + Bookings(authenticator=authenticator), + Clients(authenticator=authenticator), + Events(authenticator=authenticator), + Holidays(authenticator=authenticator), + Projects(authenticator=authenticator), + Resources(authenticator=authenticator), + ] diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json new file mode 100644 index 0000000000000..aa061822e3221 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json @@ -0,0 +1,17 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubplanner", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Hubplanner Spec", + "type": "object", + "required": ["api_key"], + "additionalProperties": true, + "properties": { + "api_key": { + "type": "string", + "description": "Hubplanner API key. See https://github.com/hubplanner/API#authentication for more details.", + "airbyte_secret": true + } + } + } +} diff --git a/airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py b/airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py new file mode 100644 index 0000000000000..3637179245002 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_hubplanner.source import SourceHubplanner + + +def test_streams(mocker): + source = SourceHubplanner() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 7 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py new file mode 100644 index 0000000000000..808f40802b20c --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py @@ -0,0 +1,79 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_hubplanner.source import HubplannerStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(HubplannerStream, "path", "v0/example_endpoint") + mocker.patch.object(HubplannerStream, "primary_key", "test_primary_key") + mocker.patch.object(HubplannerStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = HubplannerStream() + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = HubplannerStream() + inputs = {"response": MagicMock()} + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = HubplannerStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = HubplannerStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = HubplannerStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = HubplannerStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = HubplannerStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/sources/hubplanner.md b/docs/integrations/sources/hubplanner.md new file mode 100644 index 0000000000000..916e7dbd8fa35 --- /dev/null +++ b/docs/integrations/sources/hubplanner.md @@ -0,0 +1,42 @@ +# Hubplanner + +Hubplanner is a tool to plan, schedule, report and manage your entire team. + +## Prerequisites +* Create the API Key to access your data in Hubplanner. + +## Airbyte OSS +* API Key + +## Airbyte Cloud +* Comming Soon. + + +## Setup guide +### For Airbyte OSS: + +1. Access https://.hubplanner.com/settings#api or access the panel in left side Integrations/Hub Planner API +2. Click in Generate Key + +## Supported sync modes + +The Okta source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + - Full Refresh + +## Supported Streams + +- [Billing Rates](https://github.com/hubplanner/API/blob/master/Sections/billingrate.md) +- [Bookings](https://github.com/hubplanner/API/blob/master/Sections/bookings.md) +- [Clients](https://github.com/hubplanner/API/blob/master/Sections/clients.md) +- [Events](https://github.com/hubplanner/API/blob/master/Sections/events.md) +- [Holidays](https://github.com/hubplanner/API/blob/master/Sections/holidays.md) +- [Projects](https://github.com/hubplanner/API/blob/master/Sections/project.md) +- [Resources](https://github.com/hubplanner/API/blob/master/Sections/resource.md) + + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------| + +| 0.1.0 | 2021-08-10 | [12145](https://github.com/airbytehq/airbyte/pull/12145) | Initial Release |