From 90a55a6b9005ca252256f7165545144362c0dcd2 Mon Sep 17 00:00:00 2001 From: Daniel Luftspring Date: Mon, 15 Nov 2021 17:20:27 -0500 Subject: [PATCH 01/17] Connector working through read method for hellobaton source --- .../source-hellobaton/.dockerignore | 7 + .../connectors/source-hellobaton/Dockerfile | 38 + .../connectors/source-hellobaton/README.md | 132 +++ .../acceptance-test-config.yml | 30 + .../acceptance-test-docker.sh | 16 + .../connectors/source-hellobaton/build.gradle | 9 + .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 + .../integration_tests/catalog.json | 39 + .../integration_tests/configured_catalog.json | 22 + .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../connectors/source-hellobaton/main.py | 13 + .../source-hellobaton/requirements.txt | 2 + .../sample_files/configured_catalog.json | 1036 +++++++++++++++++ .../sample_files/sample_config.json | 4 + .../connectors/source-hellobaton/setup.py | 29 + .../source_hellobaton/__init__.py | 8 + .../source_hellobaton/schemas/TODO.md | 25 + .../source_hellobaton/schemas/companies.json | 30 + .../source_hellobaton/schemas/milestones.json | 92 ++ .../source_hellobaton/schemas/phases.json | 24 + .../schemas/project_attachments.json | 39 + .../source_hellobaton/schemas/projects.json | 137 +++ .../schemas/task_attachments.json | 46 + .../source_hellobaton/schemas/tasks.json | 113 ++ .../source_hellobaton/schemas/templates.json | 93 ++ .../schemas/time_entries.json | 57 + .../source_hellobaton/schemas/users.json | 46 + .../source_hellobaton/source.py | 99 ++ .../source_hellobaton/spec.json | 22 + .../source_hellobaton/streams.py | 191 +++ .../source-hellobaton/unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 59 + .../unit_tests/test_source.py | 22 + .../unit_tests/test_streams.py | 83 ++ 38 files changed, 2601 insertions(+) create mode 100644 airbyte-integrations/connectors/source-hellobaton/.dockerignore create mode 100644 airbyte-integrations/connectors/source-hellobaton/Dockerfile create mode 100644 airbyte-integrations/connectors/source-hellobaton/README.md create mode 100644 airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-hellobaton/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-hellobaton/build.gradle create mode 100644 airbyte-integrations/connectors/source-hellobaton/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-hellobaton/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-hellobaton/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/main.py create mode 100644 airbyte-integrations/connectors/source-hellobaton/requirements.txt create mode 100644 airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/sample_files/sample_config.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/setup.py create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/__init__.py create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/spec.json create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py create mode 100644 airbyte-integrations/connectors/source-hellobaton/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-hellobaton/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-hellobaton/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-hellobaton/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-hellobaton/.dockerignore b/airbyte-integrations/connectors/source-hellobaton/.dockerignore new file mode 100644 index 0000000000000..2e018bfa07087 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_hellobaton +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-hellobaton/Dockerfile b/airbyte-integrations/connectors/source-hellobaton/Dockerfile new file mode 100644 index 0000000000000..f8bf6c037c6e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_hellobaton ./source_hellobaton + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-hellobaton diff --git a/airbyte-integrations/connectors/source-hellobaton/README.md b/airbyte-integrations/connectors/source-hellobaton/README.md new file mode 100644 index 0000000000000..1b3bc774802a2 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/README.md @@ -0,0 +1,132 @@ +# Hellobaton Source + +This is the repository for the Hellobaton source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/hellobaton). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-hellobaton:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/hellobaton) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hellobaton/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source hellobaton test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-hellobaton:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-hellobaton:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-hellobaton:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hellobaton:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hellobaton:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-hellobaton:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-hellobaton:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-hellobaton:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml new file mode 100644 index 0000000000000..5c5f65ab35b52 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-hellobaton:dev +tests: + spec: + - spec_path: "source_hellobaton/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-hellobaton/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-docker.sh new file mode 100644 index 0000000000000..e4d8b1cef8961 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-hellobaton/build.gradle b/airbyte-integrations/connectors/source-hellobaton/build.gradle new file mode 100644 index 0000000000000..55c27d3615676 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_hellobaton' +} diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/__init__.py b/airbyte-integrations/connectors/source-hellobaton/integration_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..52b0f2c2118f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py new file mode 100644 index 0000000000000..58c194c5d1376 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/catalog.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/catalog.json new file mode 100644 index 0000000000000..6799946a68514 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..36f0468db0d8f --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "employees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..f3732995784f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_config.json new file mode 100644 index 0000000000000..ecc4913b84c74 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_state.json new file mode 100644 index 0000000000000..3587e579822d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/main.py b/airbyte-integrations/connectors/source-hellobaton/main.py new file mode 100644 index 0000000000000..978a73078c1b6 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_hellobaton import SourceHellobaton + +if __name__ == "__main__": + source = SourceHellobaton() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-hellobaton/requirements.txt b/airbyte-integrations/connectors/source-hellobaton/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json new file mode 100644 index 0000000000000..280ce04d722d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json @@ -0,0 +1,1036 @@ +{ + "streams": [{ + "stream": { + "name": "companies", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ], + "user_defined_primary_key": [ + [ + "id" + ] + ], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "milestones", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "project": { + "type": "string" + }, + "task_list": { + "type": "string" + }, + "phase": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + }, + "deadline_fixed": { + "type": "boolean" + }, + "deadline_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "risk_profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "risk_level": { + "type": "string" + }, + "formula": { + "type": "string" + }, + "over_run": { + "type": "integer" + } + } + } + }, + "start_datetime": { + "type": "string" + }, + "finish_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "created_from": { + "type": [ + "string", + "null" + ] + }, + "duration": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ], + "user_defined_primary_key": [ + [ + "id" + ] + ], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "projects", + "json_schema": { + "properties": { + "_self": { + "type": "string" + }, + "annual_contract_value": { + "type": "string" + }, + "attachment_list": { + "type": "string" + }, + "client_systems": { + "type": [ + "string", + "null" + ] + }, + "companies": { + "type": "array", + "items": { + "type": "string" + } + }, + "completed_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "cost": { + "type": [ + "integer", + "null" + ] + }, + "created": { + "type": "string", + "format": "date-time" + }, + "created_from": { + "type": [ + "string", + "null" + ] + }, + "created_from_template": { + "type": [ + "string", + "null" + ] + }, + "creator": { + "type": [ + "string", + "null" + ] + }, + "deadline_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "estimated_duration": { + "type": [ + "integer", + "null" + ] + }, + "id": { + "type": "integer" + }, + "implementation_budget": { + "type": "string" + }, + "milestone_list": { + "type": "string" + }, + "modified": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "phase": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "order": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + }, + "risk_profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "risk_score": { + "type": "number" + }, + "level": { + "type": "string" + }, + "variance": { + "type": "integer" + }, + "formula": { + "type": "string" + }, + "projected_golive_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + } + } + }, + "start_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "started_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "status": { + "type": "string" + }, + "task_list": { + "type": "string" + }, + "time_entry_list": { + "type": "string" + }, + "title": { + "type": "string" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ], + "user_defined_primary_key": [ + [ + "id" + ] + ], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "phases", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "order": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ], + "user_defined_primary_key": [ + [ + "id" + ] + ], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "project_attachments", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "project": { + "type": "string" + }, + "url": { + "type": "string" + }, + "label": { + "type": [ + "string", + "null" + ] + }, + "created_by": { + "type": "string" + }, + "type": { + "type": "string" + }, + "is_sow": { + "type": "boolean" + }, + "original_filename": { + "type": [ + "string", + "null" + ] + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "datetime" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ], + "user_defined_primary_key": [ + [ + "id" + ] + ], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "tasks", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "project": { + "type": "string" + }, + "status": { + "type": "string" + }, + "dependency": { + "type": [ + "string", + "null" + ] + }, + "start_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "due_datetime": { + "type": "string", + "format": "date-time" + }, + "started_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "finished_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "started_overridden_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "finished_overridden_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "estimated_duration": { + "type": [ + "integer", + "null" + ] + }, + "milestone": { + "type": "string" + }, + "created_by": { + "type": [ + "string", + "null" + ] + }, + "assigned_to": { + "type": [ + "string", + "null" + ] + }, + "created_from": { + "type": [ + "string", + "null" + ] + }, + "risk_profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "risk_level": { + "type": "string" + }, + "formula": { + "type": "string" + }, + "over_run": { + "type": "integer" + }, + "task_variance": { + "type": "integer" + }, + "cool_down": { + "type": "integer" + }, + "reason": { + "type": "integer" + }, + "duration": { + "type": "integer" + }, + "estimated_duration": { + "type": "integer" + } + } + } + }, + "time_entry_list": { + "type": "string" + }, + "attachment_list": { + "type": "string" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ], + "user_defined_primary_key": [ + [ + "id" + ] + ], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "task_attachments", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "task": { + "type": "string" + }, + "url": { + "type": "string" + }, + "type": { + "type": "string" + }, + "label": { + "type": [ + "string", + "null" + ] + }, + "deliverable": { + "type": "boolean" + }, + "requires_approval": { + "type": "boolean" + }, + "approved": { + "type": [ + "boolean", + "null" + ] + }, + "revision_task": { + "type": [ + "string", + "null" + ] + }, + "original_filename": { + "type": [ + "string", + "null" + ] + }, + "created_by": { + "type": [ + "string", + "null" + ] + }, + "created": { + "type": "string" + }, + "modified": { + "type": "string" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ], + "user_defined_primary_key": [ + [ + "id" + ] + ], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "templates", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "title": { + "type": "string" + }, + "status": { + "type": "string" + }, + "cost": { + "type": [ + "string", + "null" + ] + }, + "annual_contract_value": { + "type": [ + "string", + "null" + ] + }, + "implementation_budget": { + "type": [ + "string", + "null" + ] + }, + "estimated_duration": { + "type": [ + "integer", + "null" + ] + }, + "created_from_template": { + "type": [ + "string", + "null" + ] + }, + "created_from": { + "type": [ + "string", + "null" + ] + }, + "start_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "started_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "deadline_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "completed_datetime": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "client_systems": { + "type": [ + "string", + "null" + ] + }, + "phase": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "order": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + }, + "creator": { + "type": [ + "string", + "null" + ] + }, + "task_list": { + "type": "string" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ], + "user_defined_primary_key": [ + [ + "id" + ] + ], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "time_entries", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "user": { + "type": "string" + }, + "created_by": { + "type": "string" + }, + "project": { + "type": "string" + }, + "task": { + "type": [ + "string", + "null" + ] + }, + "rate": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "hourly_rate": { + "type": "string" + } + } + }, + "started_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "ended_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "reference_date": { + "type": "string", + "format": "date-time" + }, + "billable": { + "type": "boolean" + }, + "calculated_duration": { + "type": [ + "integer", + "null" + ] + }, + "inputted_duration": { + "type": [ + "integer", + "null" + ] + }, + "notes": { + "type": "string" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ], + "user_defined_primary_key": [ + [ + "id" + ] + ], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "account_type": { + "type": "string" + }, + "job_title": { + "type": "string" + }, + "company": { + "type": "string" + }, + "avatar_url": { + "type": [ + "string", + "null" + ] + }, + "created_by": { + "type": [ + "string", + "null" + ] + }, + "signed_up_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ], + "user_defined_primary_key": [ + [ + "id" + ] + ], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/sample_files/sample_config.json b/airbyte-integrations/connectors/source-hellobaton/sample_files/sample_config.json new file mode 100644 index 0000000000000..64990db9d07e0 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/sample_files/sample_config.json @@ -0,0 +1,4 @@ +{ + "company": "google" + "api_key": "" +} diff --git a/airbyte-integrations/connectors/source-hellobaton/setup.py b/airbyte-integrations/connectors/source-hellobaton/setup.py new file mode 100644 index 0000000000000..eda15c9efb4b2 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_hellobaton", + description="Source implementation for Hellobaton.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/__init__.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/__init__.py new file mode 100644 index 0000000000000..9137ca503a673 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceHellobaton + +__all__ = ["SourceHellobaton"] diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/TODO.md b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/TODO.md new file mode 100644 index 0000000000000..cf1efadb3c9c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json new file mode 100644 index 0000000000000..a7b2919ccbc31 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json @@ -0,0 +1,30 @@ +{ + "properties": { + + "id": { + "type": "integer" + }, + + "_self": { + "type": "string" + }, + + "name": { + "type": "string" + }, + + "type": { + "type": "string" + }, + + "created": { + "type": "string", + "format": "date-time" + }, + + "modified": { + "type": "string", + "format": "date-time" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json new file mode 100644 index 0000000000000..52e48127c7204 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json @@ -0,0 +1,92 @@ +{ + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": ["string", "null"] + }, + "project": { + "type": "string" + }, + "task_list": { + "type": "string" + }, + "phase": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + }, + "deadline_fixed": { + "type": "boolean" + }, + "deadline_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "risk_profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "risk_level": { + "type": "string" + }, + "formula": { + "type": "string" + }, + "over_run": { + "type": "integer" + } + } + } + }, + "start_datetime": { + "type": "string" + }, + "finish_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "created_from": { + "type": ["string", "null"] + }, + "duration": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json new file mode 100644 index 0000000000000..08ba90e03c77b --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json @@ -0,0 +1,24 @@ +{ + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "name": { + "type": ["string", "null"] + }, + "order": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json new file mode 100644 index 0000000000000..c6fa34c7a4499 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json @@ -0,0 +1,39 @@ +{ + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "project": { + "type": "string" + }, + "url": { + "type": "string" + }, + "label": { + "type": ["string", "null"] + }, + "created_by": { + "type": "string" + }, + "type": { + "type": "string" + }, + "is_sow": { + "type": "boolean" + }, + "original_filename": { + "type": ["string", "null"] + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "datetime" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json new file mode 100644 index 0000000000000..ca4dd9aa3dc87 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json @@ -0,0 +1,137 @@ +{ + "properties": { + + "_self": { + "type": "string" + }, + "annual_contract_value": { + "type": "string" + }, + "attachment_list": { + "type": "string" + }, + "client_systems": { + "type": ["string", "null"] + }, + "companies": { + "type": "array", + "items": { + "type": "string" + } + }, + "completed_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "cost": { + "type": ["integer", "null"] + }, + "created": { + "type": "string", + "format": "date-time" + }, + "created_from": { + "type": ["string", "null"] + }, + "created_from_template": { + "type": ["string", "null"] + }, + "creator": { + "type": ["string", "null"] + }, + "deadline_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "estimated_duration": { + "type": ["integer", "null"] + }, + "id": { + "type": "integer" + }, + "implementation_budget": { + "type": "string" + }, + "milestone_list": { + "type": "string" + }, + "modified": { + "type": ["string", "null"], + "format": "date-time" + }, + "phase": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "name": { + "type": ["string", "null"] + }, + "order": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + }, + "risk_profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "risk_score": { + "type": "number" + }, + "level": { + "type": "string" + }, + "variance": { + "type": "integer" + }, + "formula": { + "type": "string" + }, + "projected_golive_datetime": { + "type": ["string", "null"], + "format": "date-time" + } + } + } + }, + "start_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "started_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "status": { + "type": "string" + }, + "task_list": { + "type": "string" + }, + "time_entry_list": { + "type": "string" + }, + "title": { + "type": "string" + } + + } + +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json new file mode 100644 index 0000000000000..8ed7bcd2f5c04 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "task": { + "type": "string" + }, + "url": { + "type": "string" + }, + "type": { + "type": "string" + }, + "label": { + "type": ["string", "null"] + }, + "deliverable": { + "type": "boolean" + }, + "requires_approval": { + "type": "boolean" + }, + "approved": { + "type": ["boolean", "null"] + }, + "revision_task": { + "type": ["string", "null"] + }, + "original_filename": { + "type": ["string", "null"] + }, + "created_by": { + "type": ["string", "null"] + }, + "created": { + "type": "string" + }, + "modified": { + "type": "string" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json new file mode 100644 index 0000000000000..6a89c637e59d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json @@ -0,0 +1,113 @@ +{ + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": ["string", "null"] + }, + "project": { + "type": "string" + }, + "status": { + "type": "string" + }, + "dependency": { + "type": ["string", "null"] + }, + "start_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "due_datetime": { + "type": "string", + "format": "date-time" + }, + "started_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "finished_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "started_overridden_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "finished_overridden_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "estimated_duration": { + "type": ["integer", "null"] + }, + "milestone": { + "type": "string" + }, + "created_by": { + "type": ["string", "null"] + }, + "assigned_to": { + "type": ["string", "null"] + }, + "created_from": { + "type": ["string", "null"] + }, + "risk_profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "risk_level": { + "type": "string" + }, + "formula": { + "type": "string" + }, + "over_run": { + "type": "integer" + }, + "task_variance": { + "type": "integer" + }, + "cool_down": { + "type": "integer" + }, + "reason": { + "type": "integer" + }, + "duration": { + "type": "integer" + }, + "estimated_duration": { + "type": "integer" + } + } + } + }, + "time_entry_list": { + "type": "string" + }, + "attachment_list": { + "type": "string" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json new file mode 100644 index 0000000000000..cc317d9ee1c75 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json @@ -0,0 +1,93 @@ +{ + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "title": { + "type": "string" + }, + "status": { + "type": "string" + }, + "cost": { + "type": [ + "string", + "null" + ] + }, + "annual_contract_value": { + "type": [ + "string", + "null" + ] + }, + "implementation_budget": { + "type": [ + "string", + "null" + ] + }, + "estimated_duration": { + "type": [ + "integer", + "null" + ] + }, + "created_from_template": { + "type": ["string", "null"] + }, + "created_from": { + "type": ["string", "null"] + }, + "start_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "started_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "deadline_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "completed_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "client_systems": { + "type": ["string", "null"] + }, + "phase": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "order": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + }, + "creator": { + "type": ["string", "null"] + }, + "task_list": { + "type": "string" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json new file mode 100644 index 0000000000000..6172a67fa3e7f --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json @@ -0,0 +1,57 @@ +{ + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "user": { + "type": "string" + }, + "created_by": { + "type": "string" + }, + "project": { + "type": "string" + }, + "task": { + "type": ["string", "null"] + }, + "rate": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "hourly_rate": { + "type": "string" + } + } + }, + "started_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "ended_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "reference_date": { + "type": "string", + "format": "date-time" + }, + "billable": { + "type": "boolean" + }, + "calculated_duration": { + "type": ["integer", "null"] + }, + "inputted_duration": { + "type": ["integer", "null"] + }, + "notes": { + "type": "string" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json new file mode 100644 index 0000000000000..7ddb75c75f159 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json @@ -0,0 +1,46 @@ +{ + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "account_type": { + "type": "string" + }, + "job_title": { + "type": "string" + }, + "company": { + "type": "string" + }, + "avatar_url": { + "type": ["string", "null"] + }, + "created_by": { + "type": ["string", "null"] + }, + "signed_up_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py new file mode 100644 index 0000000000000..d960ba006c076 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py @@ -0,0 +1,99 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.sources.streams import Stream + +from .streams import ( + HellobatonStream, + Companies, + Milestones, + Projects, + Phases, + ProjectAttachments, + Tasks, + TaskAttachments, + Templates, + TimeEntries, + Users +) + +STREAMS = [ + Companies, + Milestones, + Projects, + Phases, + ProjectAttachments, + Tasks, + TaskAttachments, + Templates, + TimeEntries, + Users +] + +#TODO ADD INCREMENTAL STATE +# # Basic incremental stream +# class IncrementalHellobatonStream(HellobatonStream, ABC): +# """ +# TODO fill in details of this class to implement functionality related to incremental syncs for your connector. +# if you do not need to implement incremental sync for any streams, remove this class. +# """ + +# # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. +# state_checkpoint_interval = None + +# @property +# def cursor_field(self) -> str: +# """ +# TODO +# Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is +# usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + +# :return str: The name of the cursor field. +# """ +# return [] + +# def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: +# """ +# Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and +# the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. +# """ +# return {} + + +# Source +class SourceHellobaton(AbstractSource): + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, any]) -> Tuple[bool, any]: + """ + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + try: + params = { + "api_key": config["api_key"], + } + base_url = HellobatonStream.url_template.format(company=config["company"]) + #This is just going to return a mapping of available endpoints + response = requests.get(base_url, params=params) + status_code = response.status_code + logger.info(f"Status code: {status_code}") + if status_code == 200: + return True, None + + except Exception as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + return [stream(company=config["company"], api_key=config["api_key"]) for stream in STREAMS] diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/spec.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/spec.json new file mode 100644 index 0000000000000..3f286953ec5ec --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/spec.json @@ -0,0 +1,22 @@ +{ + "documentationUrl": "https://docsurl.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Hellobaton Spec", + "type": "object", + "required": ["api_key", "company"], + "additionalProperties": false, + "properties": { + "api_key": { + "type": "string", + "description": "authentication key required to access the api endpoints", + "airbyte_secret": true + }, + "company": { + "type": "string", + "description": "Company name that generates your base api url", + "examples": ["google", "facebook", "microsoft"] + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py new file mode 100644 index 0000000000000..620094958c4b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py @@ -0,0 +1,191 @@ + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from urllib.parse import ( + urlparse, + parse_qs +) +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +# Basic full refresh stream +class HellobatonStream(HttpStream, ABC): + """ + This class represents a stream output by the connector. + This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, + parsing responses etc.. + """ + + page_size: int = 100 + primary_key: str = "id" + + def __init__(self, company: str, api_key: str, **kwargs): + super().__init__(**kwargs) + self.api_key = api_key + self.company = company + + @property + def url_base(self) -> str: + """ + Using this method instead of class init to dynamically generate base url based on config + """ + company=self.company + return f"https://{company}.hellobaton.com/api/" + + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + Logic to generate next page token based on the response + """ + + payload = response.json() + result_count = payload['count'] + + if result_count > self.page_size: + query_string = urlparse(payload['next']).query + next_page_token = parse_qs(query_string).get('page', None) + + else: + next_page_token = None + + return next_page_token + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + API request params which expect an api key for auth and any pagination is done using defined in the next_page_token method + """ + + params = { + 'api_key': self.api_key, + 'page_size': self.page_size, + 'page': next_page_token + } + + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + May want to add logic here to unpack foreign keys from urls but tbd + For now each response record is accessed through the results key in the JSON payload + """ + for results in response.json()['results']: + yield results + + +class Companies(HellobatonStream): + """ + Companies stream class + """ + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + + return "companies" + +class Milestones(HellobatonStream): + """ + Milestones stream class + """ + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + + return "milestones" + +class Phases(HellobatonStream): + """ + Phases stream class + """ + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + + return "phases" + + +class ProjectAttachments(HellobatonStream): + """ + Project attachments stream class + """ + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + + return "project_attachments" + +class Projects(HellobatonStream): + """ + Projects stream class + """ + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + + return "projects" + +class Tasks(HellobatonStream): + """ + Tasks stream class + """ + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + + return "tasks" + +class TaskAttachments(HellobatonStream): + """ + Task attachments stream class + """ + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + + return "task_attachments" + + +class Templates(HellobatonStream): + """ + Templates stream class + """ + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + + return "templates" + +class TimeEntries(HellobatonStream): + """ + Time entries stream class + """ + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + + return "time_entries" + +class Users(HellobatonStream): + """ + Users stream class + """ + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + + return "users" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/unit_tests/__init__.py b/airbyte-integrations/connectors/source-hellobaton/unit_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_incremental_streams.py new file mode 100644 index 0000000000000..a955d9f973b91 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_hellobaton.source import IncrementalHellobatonStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalHellobatonStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalHellobatonStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalHellobatonStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalHellobatonStream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalHellobatonStream() + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalHellobatonStream() + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalHellobatonStream, "cursor_field", "dummy_field") + stream = IncrementalHellobatonStream() + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalHellobatonStream() + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalHellobatonStream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_source.py new file mode 100644 index 0000000000000..852c8658a0fad --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_hellobaton.source import SourceHellobaton + + +def test_check_connection(mocker): + source = SourceHellobaton() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceHellobaton() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_streams.py new file mode 100644 index 0000000000000..a1fa1eb75aef4 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_hellobaton.source import HellobatonStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(HellobatonStream, "path", "v0/example_endpoint") + mocker.patch.object(HellobatonStream, "primary_key", "test_primary_key") + mocker.patch.object(HellobatonStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = HellobatonStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = HellobatonStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = HellobatonStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = HellobatonStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = HellobatonStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = HellobatonStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = HellobatonStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time From 2478ceee8059aab3be7ef47ba3e86cf26d7262c5 Mon Sep 17 00:00:00 2001 From: Daniel Luftspring Date: Tue, 30 Nov 2021 09:49:31 -0500 Subject: [PATCH 02/17] Check connection method fixes --- .../connectors/source-hellobaton/source_hellobaton/source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py index d960ba006c076..970de4c764812 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py @@ -76,11 +76,12 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, any]) -> :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ + url_template="https://{company}.hellobaton.com/api/" try: params = { "api_key": config["api_key"], } - base_url = HellobatonStream.url_template.format(company=config["company"]) + base_url = url_template.format(company=config["company"]) #This is just going to return a mapping of available endpoints response = requests.get(base_url, params=params) status_code = response.status_code From c63c4901d0f551a318609b6030c5d20242ce2efb Mon Sep 17 00:00:00 2001 From: Daniel Luftspring Date: Wed, 1 Dec 2021 13:40:14 -0500 Subject: [PATCH 03/17] Cleaning up directories and files +activity stream --- .../acceptance-test-config.yml | 10 --- .../sample_files/configured_catalog.json | 66 +++++++++++++++ .../source_hellobaton/schemas/TODO.md | 25 ------ .../source_hellobaton/schemas/activity.json | 45 ++++++++++ .../source_hellobaton/source.py | 32 +------ .../source_hellobaton/streams.py | 11 +++ .../source-hellobaton/unit_tests/__init__.py | 3 - .../unit_tests/test_incremental_streams.py | 59 ------------- .../unit_tests/test_source.py | 22 ----- .../unit_tests/test_streams.py | 83 ------------------- 10 files changed, 124 insertions(+), 232 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json delete mode 100644 airbyte-integrations/connectors/source-hellobaton/unit_tests/__init__.py delete mode 100644 airbyte-integrations/connectors/source-hellobaton/unit_tests/test_incremental_streams.py delete mode 100644 airbyte-integrations/connectors/source-hellobaton/unit_tests/test_source.py delete mode 100644 airbyte-integrations/connectors/source-hellobaton/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml index 5c5f65ab35b52..0134ff72454b5 100644 --- a/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml @@ -15,16 +15,6 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json index 280ce04d722d9..77dd54d29ac8b 100644 --- a/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json @@ -1,5 +1,71 @@ { "streams": [{ + "stream": { + "name": "activity", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "type": { + "type": ["string", "null"] + }, + "group": { + "type": "string" + }, + "parent": { + "type": ["string", "null"] + }, + "child": { + "type": ["string", "null"] + }, + "actor": { + "type": "string" + }, + "project": { + "type": "string" + }, + "parent_type": { + "type": "string" + }, + "child_type": { + "type": "string" + }, + "meta": { + "type": "string" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + } + + }, + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ], + "user_defined_primary_key": [ + [ + "id" + ] + ], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { "stream": { "name": "companies", "json_schema": { diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/TODO.md b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/TODO.md deleted file mode 100644 index cf1efadb3c9c9..0000000000000 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/TODO.md +++ /dev/null @@ -1,25 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. - -The schema of a stream is the return value of `Stream.get_json_schema`. - -## Static schemas -By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. - -Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. - -## Dynamic schemas -If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). - -## Dynamically modifying static schemas -Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: -``` -def get_json_schema(self): - schema = super().get_json_schema() - schema['dynamically_determined_property'] = "property" - return schema -``` - -Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json new file mode 100644 index 0000000000000..2935c41820356 --- /dev/null +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json @@ -0,0 +1,45 @@ +{ + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "type": { + "type": ["string", "null"] + }, + "group": { + "type": "string" + }, + "parent": { + "type": ["string", "null"] + }, + "child": { + "type": ["string", "null"] + }, + "actor": { + "type": "string" + }, + "project": { + "type": "string" + }, + "parent_type": { + "type": "string" + }, + "child_type": { + "type": "string" + }, + "meta": { + "type": "string" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py index 970de4c764812..65140b13103d7 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py @@ -13,6 +13,7 @@ from .streams import ( HellobatonStream, + Activity, Companies, Milestones, Projects, @@ -26,6 +27,7 @@ ) STREAMS = [ + Activity, Companies, Milestones, Projects, @@ -38,36 +40,6 @@ Users ] -#TODO ADD INCREMENTAL STATE -# # Basic incremental stream -# class IncrementalHellobatonStream(HellobatonStream, ABC): -# """ -# TODO fill in details of this class to implement functionality related to incremental syncs for your connector. -# if you do not need to implement incremental sync for any streams, remove this class. -# """ - -# # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. -# state_checkpoint_interval = None - -# @property -# def cursor_field(self) -> str: -# """ -# TODO -# Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is -# usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - -# :return str: The name of the cursor field. -# """ -# return [] - -# def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: -# """ -# Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and -# the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. -# """ -# return {} - - # Source class SourceHellobaton(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, any]) -> Tuple[bool, any]: diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py index 620094958c4b1..93a68709e177e 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py @@ -78,6 +78,17 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp yield results +class Activity(HellobatonStream): + """ + Activity stream class + """ + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + + return "activity" + class Companies(HellobatonStream): """ Companies stream class diff --git a/airbyte-integrations/connectors/source-hellobaton/unit_tests/__init__.py b/airbyte-integrations/connectors/source-hellobaton/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec6..0000000000000 --- a/airbyte-integrations/connectors/source-hellobaton/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_incremental_streams.py deleted file mode 100644 index a955d9f973b91..0000000000000 --- a/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,59 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.models import SyncMode -from pytest import fixture -from source_hellobaton.source import IncrementalHellobatonStream - - -@fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(IncrementalHellobatonStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalHellobatonStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalHellobatonStream, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class): - stream = IncrementalHellobatonStream() - # TODO: replace this with your expected cursor field - expected_cursor_field = [] - assert stream.cursor_field == expected_cursor_field - - -def test_get_updated_state(patch_incremental_base_class): - stream = IncrementalHellobatonStream() - # TODO: replace this with your input parameters - inputs = {"current_stream_state": None, "latest_record": None} - # TODO: replace this with your expected updated stream state - expected_state = {} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_stream_slices(patch_incremental_base_class): - stream = IncrementalHellobatonStream() - # TODO: replace this with your input parameters - inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} - # TODO: replace this with your expected stream slices list - expected_stream_slice = [None] - assert stream.stream_slices(**inputs) == expected_stream_slice - - -def test_supports_incremental(patch_incremental_base_class, mocker): - mocker.patch.object(IncrementalHellobatonStream, "cursor_field", "dummy_field") - stream = IncrementalHellobatonStream() - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class): - stream = IncrementalHellobatonStream() - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = IncrementalHellobatonStream() - # TODO: replace this with your expected checkpoint interval - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_source.py deleted file mode 100644 index 852c8658a0fad..0000000000000 --- a/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_source.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_hellobaton.source import SourceHellobaton - - -def test_check_connection(mocker): - source = SourceHellobaton() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceHellobaton() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_streams.py deleted file mode 100644 index a1fa1eb75aef4..0000000000000 --- a/airbyte-integrations/connectors/source-hellobaton/unit_tests/test_streams.py +++ /dev/null @@ -1,83 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_hellobaton.source import HellobatonStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(HellobatonStream, "path", "v0/example_endpoint") - mocker.patch.object(HellobatonStream, "primary_key", "test_primary_key") - mocker.patch.object(HellobatonStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = HellobatonStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = HellobatonStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected next page token - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = HellobatonStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = HellobatonStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = HellobatonStream() - # TODO: replace this with your expected http request method - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = HellobatonStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = HellobatonStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time From 3b4ae8ecf40a13bfd229664317b020312950f841 Mon Sep 17 00:00:00 2001 From: Daniel Luftspring Date: Wed, 1 Dec 2021 14:21:37 -0500 Subject: [PATCH 04/17] Changing webapp port for testing --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 31e304b9d61af..b6b66801fe35b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -165,7 +165,7 @@ services: container_name: airbyte-webapp restart: unless-stopped ports: - - 8000:80 + - 8000:6666 environment: - AIRBYTE_ROLE=${AIRBYTE_ROLE:-} - AIRBYTE_VERSION=${VERSION} From eedef370f8d7be295c1dc0265d3ba34a4b5a701b Mon Sep 17 00:00:00 2001 From: Daniel Luftspring Date: Wed, 1 Dec 2021 14:23:21 -0500 Subject: [PATCH 05/17] More testing ports --- docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index b6b66801fe35b..6cf9162f396a9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -154,7 +154,7 @@ services: - SECRET_PERSISTENCE=${SECRET_PERSISTENCE} - VERSION_0_32_0_FORCE_UPGRADE=${VERSION_0_32_0_FORCE_UPGRADE} ports: - - 8001:8001 + - 6667:6667 volumes: - workspace:${WORKSPACE_ROOT} - data:${CONFIG_ROOT} @@ -165,7 +165,7 @@ services: container_name: airbyte-webapp restart: unless-stopped ports: - - 8000:6666 + - 6666:6666 environment: - AIRBYTE_ROLE=${AIRBYTE_ROLE:-} - AIRBYTE_VERSION=${VERSION} From 3983c8eada6da7d5b8d8188700d74ce6f621163f Mon Sep 17 00:00:00 2001 From: Daniel Luftspring Date: Wed, 1 Dec 2021 14:39:09 -0500 Subject: [PATCH 06/17] Ports ports ports --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 6cf9162f396a9..ae972a388c71c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -165,7 +165,7 @@ services: container_name: airbyte-webapp restart: unless-stopped ports: - - 6666:6666 + - 6666:80 environment: - AIRBYTE_ROLE=${AIRBYTE_ROLE:-} - AIRBYTE_VERSION=${VERSION} From 2c673c027d214ffa2b3c686ea258e606449e0cc0 Mon Sep 17 00:00:00 2001 From: Daniel Luftspring Date: Wed, 1 Dec 2021 14:48:14 -0500 Subject: [PATCH 07/17] porto --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index ae972a388c71c..6cf9162f396a9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -165,7 +165,7 @@ services: container_name: airbyte-webapp restart: unless-stopped ports: - - 6666:80 + - 6666:6666 environment: - AIRBYTE_ROLE=${AIRBYTE_ROLE:-} - AIRBYTE_VERSION=${VERSION} From 56b478a9d0b450c1bcd54c66dba6bdccb51a6582 Mon Sep 17 00:00:00 2001 From: Daniel Luftspring Date: Thu, 2 Dec 2021 10:26:55 -0500 Subject: [PATCH 08/17] Reverting port changes --- docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 6cf9162f396a9..31e304b9d61af 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -154,7 +154,7 @@ services: - SECRET_PERSISTENCE=${SECRET_PERSISTENCE} - VERSION_0_32_0_FORCE_UPGRADE=${VERSION_0_32_0_FORCE_UPGRADE} ports: - - 6667:6667 + - 8001:8001 volumes: - workspace:${WORKSPACE_ROOT} - data:${CONFIG_ROOT} @@ -165,7 +165,7 @@ services: container_name: airbyte-webapp restart: unless-stopped ports: - - 6666:6666 + - 8000:80 environment: - AIRBYTE_ROLE=${AIRBYTE_ROLE:-} - AIRBYTE_VERSION=${VERSION} From 19235f7165b57e7b2eadba1bfbe6f449e52a0f74 Mon Sep 17 00:00:00 2001 From: Daniel Luftspring Date: Sun, 12 Dec 2021 11:34:35 -0500 Subject: [PATCH 09/17] Updating activity stream schema and config catalog --- .../source-hellobaton/sample_files/configured_catalog.json | 2 +- .../source-hellobaton/source_hellobaton/schemas/activity.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json index 77dd54d29ac8b..25e89efb32895 100644 --- a/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json @@ -35,7 +35,7 @@ "type": "string" }, "meta": { - "type": "string" + "type": ["object", "null"] }, "created": { "type": "string", diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json index 2935c41820356..333574f3bc86d 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json @@ -31,7 +31,7 @@ "type": "string" }, "meta": { - "type": "string" + "type": ["object", "null"] }, "created": { "type": "string", From ce0ad6225a31f2532e470b13298381898bf8fe05 Mon Sep 17 00:00:00 2001 From: Daniel Luftspring Date: Thu, 16 Dec 2021 09:10:50 -0500 Subject: [PATCH 10/17] Fixing flake lint failures --- .../integration_tests/acceptance.py | 2 +- .../source_hellobaton/source.py | 33 +++------ .../source_hellobaton/streams.py | 68 ++++++++++--------- 3 files changed, 46 insertions(+), 57 deletions(-) diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py index 58c194c5d1376..056971f954502 100644 --- a/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/acceptance.py @@ -10,7 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py index 65140b13103d7..1c6edc6479c45 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/source.py @@ -3,42 +3,29 @@ # -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, List, Mapping, Tuple import requests -from airbyte_cdk.sources import AbstractSource from airbyte_cdk import AirbyteLogger +from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from .streams import ( - HellobatonStream, Activity, Companies, Milestones, - Projects, Phases, ProjectAttachments, - Tasks, + Projects, TaskAttachments, + Tasks, Templates, TimeEntries, - Users + Users, ) -STREAMS = [ - Activity, - Companies, - Milestones, - Projects, - Phases, - ProjectAttachments, - Tasks, - TaskAttachments, - Templates, - TimeEntries, - Users -] +STREAMS = [Activity, Companies, Milestones, Projects, Phases, ProjectAttachments, Tasks, TaskAttachments, Templates, TimeEntries, Users] + # Source class SourceHellobaton(AbstractSource): @@ -48,13 +35,13 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, any]) -> :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - url_template="https://{company}.hellobaton.com/api/" + url_template = "https://{company}.hellobaton.com/api/" try: params = { "api_key": config["api_key"], - } + } base_url = url_template.format(company=config["company"]) - #This is just going to return a mapping of available endpoints + # This is just going to return a mapping of available endpoints response = requests.get(base_url, params=params) status_code = response.status_code logger.info(f"Status code: {status_code}") diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py index 93a68709e177e..9b36eab622041 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/streams.py @@ -1,16 +1,15 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, Iterable, Mapping, MutableMapping, Optional +from urllib.parse import parse_qs, urlparse import requests -from urllib.parse import ( - urlparse, - parse_qs -) -from airbyte_cdk import AirbyteLogger from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + # Basic full refresh stream class HellobatonStream(HttpStream, ABC): @@ -27,28 +26,27 @@ def __init__(self, company: str, api_key: str, **kwargs): super().__init__(**kwargs) self.api_key = api_key self.company = company - + @property def url_base(self) -> str: """ Using this method instead of class init to dynamically generate base url based on config """ - company=self.company + company = self.company return f"https://{company}.hellobaton.com/api/" - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ Logic to generate next page token based on the response """ payload = response.json() - result_count = payload['count'] + result_count = payload["count"] if result_count > self.page_size: - query_string = urlparse(payload['next']).query - next_page_token = parse_qs(query_string).get('page', None) - + query_string = urlparse(payload["next"]).query + next_page_token = parse_qs(query_string).get("page", None) + else: next_page_token = None @@ -61,11 +59,7 @@ def request_params( API request params which expect an api key for auth and any pagination is done using defined in the next_page_token method """ - params = { - 'api_key': self.api_key, - 'page_size': self.page_size, - 'page': next_page_token - } + params = {"api_key": self.api_key, "page_size": self.page_size, "page": next_page_token} return params @@ -74,7 +68,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp May want to add logic here to unpack foreign keys from urls but tbd For now each response record is accessed through the results key in the JSON payload """ - for results in response.json()['results']: + for results in response.json()["results"]: yield results @@ -86,9 +80,10 @@ class Activity(HellobatonStream): def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - + return "activity" + class Companies(HellobatonStream): """ Companies stream class @@ -97,9 +92,10 @@ class Companies(HellobatonStream): def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - + return "companies" + class Milestones(HellobatonStream): """ Milestones stream class @@ -108,9 +104,10 @@ class Milestones(HellobatonStream): def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - + return "milestones" + class Phases(HellobatonStream): """ Phases stream class @@ -119,7 +116,7 @@ class Phases(HellobatonStream): def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - + return "phases" @@ -131,9 +128,10 @@ class ProjectAttachments(HellobatonStream): def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - + return "project_attachments" + class Projects(HellobatonStream): """ Projects stream class @@ -142,9 +140,10 @@ class Projects(HellobatonStream): def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - + return "projects" + class Tasks(HellobatonStream): """ Tasks stream class @@ -153,9 +152,10 @@ class Tasks(HellobatonStream): def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - + return "tasks" + class TaskAttachments(HellobatonStream): """ Task attachments stream class @@ -164,7 +164,7 @@ class TaskAttachments(HellobatonStream): def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - + return "task_attachments" @@ -176,9 +176,10 @@ class Templates(HellobatonStream): def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - + return "templates" + class TimeEntries(HellobatonStream): """ Time entries stream class @@ -187,9 +188,10 @@ class TimeEntries(HellobatonStream): def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - + return "time_entries" + class Users(HellobatonStream): """ Users stream class @@ -198,5 +200,5 @@ class Users(HellobatonStream): def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - - return "users" \ No newline at end of file + + return "users" From b6f82429327031af9d786ee334ca3a8566859f9b Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Thu, 13 Jan 2022 22:19:51 -0300 Subject: [PATCH 11/17] fix schemas --- .vscode/settings.json | 7 ++++ .../integration_tests/catalog.json | 36 ++----------------- .../integration_tests/configured_catalog.json | 11 +----- .../integration_tests/invalid_config.json | 5 +-- .../integration_tests/sample_config.json | 5 +-- .../source_hellobaton/schemas/activity.json | 3 ++ .../source_hellobaton/schemas/projects.json | 3 ++ .../source_hellobaton/schemas/tasks.json | 3 ++ 8 files changed, 26 insertions(+), 47 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000000..4f42e99d77bca --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "python.testing.pytestArgs": [ + "airbyte-integrations" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/catalog.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/catalog.json index 6799946a68514..db86d334a63d8 100644 --- a/airbyte-integrations/connectors/source-hellobaton/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/catalog.json @@ -1,39 +1,9 @@ { "streams": [ { - "name": "TODO fix this file", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": "column1", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - }, - { - "name": "table1", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } + "name": "tasks", + "supported_sync_modes": ["full_refresh"], + "json_schema": {} } ] } diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/configured_catalog.json index 36f0468db0d8f..e111623b73a3d 100644 --- a/airbyte-integrations/connectors/source-hellobaton/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/configured_catalog.json @@ -2,21 +2,12 @@ "streams": [ { "stream": { - "name": "customers", + "name": "projects", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "employees", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/invalid_config.json index f3732995784f2..b0b4a7aba0734 100644 --- a/airbyte-integrations/connectors/source-hellobaton/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/invalid_config.json @@ -1,3 +1,4 @@ { - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" -} + "api_key": "invalid-api-key", + "company": "non-valid" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_config.json index ecc4913b84c74..b0b4a7aba0734 100644 --- a/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_config.json @@ -1,3 +1,4 @@ { - "fix-me": "TODO" -} + "api_key": "invalid-api-key", + "company": "non-valid" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json index 333574f3bc86d..9f03549263216 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json @@ -1,4 +1,7 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json index ca4dd9aa3dc87..46add2d1362ab 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json @@ -1,4 +1,7 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "_self": { diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json index 6a89c637e59d0..c6122b86eb0df 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json @@ -1,4 +1,7 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" From ff1218a6671ce81072b6f02e7f04e0ae8fd17528 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Thu, 13 Jan 2022 22:41:30 -0300 Subject: [PATCH 12/17] correct schemas and acceptance tests --- .vscode/settings.json | 7 -- .../acceptance-test-config.yml | 2 +- .../integration_tests/configured_catalog.json | 90 +++++++++++++++++++ .../sample_files/sample_config.json | 2 +- .../source_hellobaton/schemas/activity.json | 8 +- .../source_hellobaton/schemas/companies.json | 3 + .../source_hellobaton/schemas/milestones.json | 3 + .../source_hellobaton/schemas/phases.json | 3 + .../schemas/project_attachments.json | 3 + .../schemas/task_attachments.json | 3 + .../source_hellobaton/schemas/templates.json | 3 + .../schemas/time_entries.json | 3 + .../source_hellobaton/schemas/users.json | 3 + 13 files changed, 120 insertions(+), 13 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 4f42e99d77bca..0000000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "python.testing.pytestArgs": [ - "airbyte-integrations" - ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml index 0134ff72454b5..79787e5b96207 100644 --- a/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-config.yml @@ -14,7 +14,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + empty_streams: ["templates", "time_entries"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/configured_catalog.json index e111623b73a3d..8261b74756911 100644 --- a/airbyte-integrations/connectors/source-hellobaton/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/configured_catalog.json @@ -1,5 +1,50 @@ { "streams": [ + { + "stream": { + "name": "activity", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "companies", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "milestones", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "phases", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "project_attachments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "projects", @@ -8,6 +53,51 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "task_attachments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "tasks", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "templates", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "time_entries", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-hellobaton/sample_files/sample_config.json b/airbyte-integrations/connectors/source-hellobaton/sample_files/sample_config.json index 64990db9d07e0..312a9dfadae27 100644 --- a/airbyte-integrations/connectors/source-hellobaton/sample_files/sample_config.json +++ b/airbyte-integrations/connectors/source-hellobaton/sample_files/sample_config.json @@ -1,4 +1,4 @@ { - "company": "google" + "company": "google", "api_key": "" } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json index 9f03549263216..c21d941a29be1 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json @@ -22,16 +22,16 @@ "type": ["string", "null"] }, "actor": { - "type": "string" + "type": ["string", "null"] }, "project": { - "type": "string" + "type": ["string", "null"] }, "parent_type": { - "type": "string" + "type": ["string", "null"] }, "child_type": { - "type": "string" + "type": ["string", "null"] }, "meta": { "type": ["object", "null"] diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json index a7b2919ccbc31..11efb5fbbec6c 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json @@ -1,4 +1,7 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json index 52e48127c7204..1892900ff5f39 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json @@ -1,4 +1,7 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json index 08ba90e03c77b..c1d915ea88c37 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json @@ -1,4 +1,7 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json index c6fa34c7a4499..256dab1f8ee38 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json @@ -1,4 +1,7 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json index 8ed7bcd2f5c04..0ad60ebfa2d59 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json @@ -1,4 +1,7 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json index cc317d9ee1c75..d5335f9292908 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json @@ -1,4 +1,7 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json index 6172a67fa3e7f..76af78e44d0d1 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json @@ -1,4 +1,7 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json index 7ddb75c75f159..76faf58fedfc1 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json @@ -1,4 +1,7 @@ { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" From 305fe2a216f27e19defbf708ceb79c2029bd48c3 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Thu, 13 Jan 2022 22:54:42 -0300 Subject: [PATCH 13/17] run format --- .../integration_tests/invalid_config.json | 2 +- .../integration_tests/sample_config.json | 2 +- .../sample_files/configured_catalog.json | 1879 +++++++---------- .../sample_files/sample_config.json | 4 +- .../source_hellobaton/schemas/activity.json | 92 +- .../source_hellobaton/schemas/companies.json | 51 +- .../source_hellobaton/schemas/milestones.json | 166 +- .../source_hellobaton/schemas/phases.json | 50 +- .../schemas/project_attachments.json | 80 +- .../source_hellobaton/schemas/projects.json | 249 ++- .../schemas/task_attachments.json | 94 +- .../source_hellobaton/schemas/tasks.json | 2 +- .../source_hellobaton/schemas/templates.json | 162 +- .../schemas/time_entries.json | 110 +- .../source_hellobaton/schemas/users.json | 94 +- .../source_hellobaton/spec.json | 40 +- 16 files changed, 1387 insertions(+), 1690 deletions(-) diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/invalid_config.json index b0b4a7aba0734..792b2d6ed172b 100644 --- a/airbyte-integrations/connectors/source-hellobaton/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/invalid_config.json @@ -1,4 +1,4 @@ { "api_key": "invalid-api-key", "company": "non-valid" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_config.json index b0b4a7aba0734..792b2d6ed172b 100644 --- a/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-hellobaton/integration_tests/sample_config.json @@ -1,4 +1,4 @@ { "api_key": "invalid-api-key", "company": "non-valid" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json index 25e89efb32895..0a6393a2c4ecf 100644 --- a/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-hellobaton/sample_files/configured_catalog.json @@ -1,1102 +1,815 @@ { - "streams": [{ - "stream": { - "name": "activity", - "json_schema": { - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "type": { - "type": ["string", "null"] - }, - "group": { - "type": "string" - }, - "parent": { - "type": ["string", "null"] - }, - "child": { - "type": ["string", "null"] - }, - "actor": { - "type": "string" - }, - "project": { - "type": "string" - }, - "parent_type": { - "type": "string" - }, - "child_type": { - "type": "string" - }, - "meta": { - "type": ["object", "null"] - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } - } - } - - }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ], - "user_defined_primary_key": [ - [ - "id" - ] - ], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "companies", - "json_schema": { - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "name": { - "type": "string" - }, - "type": { - "type": "string" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } - } - } + "streams": [ + { + "stream": { + "name": "activity", + "json_schema": { + "properties": { + "id": { + "type": "integer" }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ], - "user_defined_primary_key": [ - [ - "id" - ] - ], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "milestones", - "json_schema": { - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "title": { - "type": "string" - }, - "description": { - "type": [ - "string", - "null" - ] - }, - "project": { - "type": "string" - }, - "task_list": { - "type": "string" - }, - "phase": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "name": { - "type": "string" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } - } - }, - "deadline_fixed": { - "type": "boolean" - }, - "deadline_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "risk_profiles": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "risk_level": { - "type": "string" - }, - "formula": { - "type": "string" - }, - "over_run": { - "type": "integer" - } - } - } - }, - "start_datetime": { - "type": "string" - }, - "finish_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "created_from": { - "type": [ - "string", - "null" - ] - }, - "duration": { - "type": "integer" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } - } - } + "_self": { + "type": "string" }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ], - "user_defined_primary_key": [ - [ - "id" - ] - ], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "projects", - "json_schema": { - "properties": { - "_self": { - "type": "string" - }, - "annual_contract_value": { - "type": "string" - }, - "attachment_list": { - "type": "string" - }, - "client_systems": { - "type": [ - "string", - "null" - ] - }, - "companies": { - "type": "array", - "items": { - "type": "string" - } - }, - "completed_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "cost": { - "type": [ - "integer", - "null" - ] - }, - "created": { - "type": "string", - "format": "date-time" - }, - "created_from": { - "type": [ - "string", - "null" - ] - }, - "created_from_template": { - "type": [ - "string", - "null" - ] - }, - "creator": { - "type": [ - "string", - "null" - ] - }, - "deadline_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "estimated_duration": { - "type": [ - "integer", - "null" - ] - }, - "id": { - "type": "integer" - }, - "implementation_budget": { - "type": "string" - }, - "milestone_list": { - "type": "string" - }, - "modified": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "phase": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "order": { - "type": "integer" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } - } - }, - "risk_profiles": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "risk_score": { - "type": "number" - }, - "level": { - "type": "string" - }, - "variance": { - "type": "integer" - }, - "formula": { - "type": "string" - }, - "projected_golive_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - } - } - } - }, - "start_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "started_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "status": { - "type": "string" - }, - "task_list": { - "type": "string" - }, - "time_entry_list": { - "type": "string" - }, - "title": { - "type": "string" - } - } - } + "type": { + "type": ["string", "null"] + }, + "group": { + "type": "string" + }, + "parent": { + "type": ["string", "null"] + }, + "child": { + "type": ["string", "null"] + }, + "actor": { + "type": "string" + }, + "project": { + "type": "string" + }, + "parent_type": { + "type": "string" + }, + "child_type": { + "type": "string" + }, + "meta": { + "type": ["object", "null"] + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "user_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "companies", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "created": { + "type": "string", + "format": "date-time" }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ], - "user_defined_primary_key": [ - [ - "id" - ] - ], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "phases", - "json_schema": { - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "name": { - "type": [ - "string", - "null" - ] - }, - "order": { - "type": "integer" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } - } + "modified": { + "type": "string", + "format": "date-time" + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "user_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "milestones", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": ["string", "null"] + }, + "project": { + "type": "string" + }, + "task_list": { + "type": "string" + }, + "phase": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "name": { + "type": "string" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" } + } + }, + "deadline_fixed": { + "type": "boolean" }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ], - "user_defined_primary_key": [ - [ - "id" - ] - ], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "project_attachments", - "json_schema": { - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "project": { - "type": "string" - }, - "url": { - "type": "string" - }, - "label": { - "type": [ - "string", - "null" - ] - }, - "created_by": { - "type": "string" - }, - "type": { - "type": "string" - }, - "is_sow": { - "type": "boolean" - }, - "original_filename": { - "type": [ - "string", - "null" - ] - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "datetime" - } - } + "deadline_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "risk_profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "risk_level": { + "type": "string" + }, + "formula": { + "type": "string" + }, + "over_run": { + "type": "integer" + } } + } + }, + "start_datetime": { + "type": "string" + }, + "finish_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "created_from": { + "type": ["string", "null"] + }, + "duration": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "user_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "projects", + "json_schema": { + "properties": { + "_self": { + "type": "string" + }, + "annual_contract_value": { + "type": "string" + }, + "attachment_list": { + "type": "string" + }, + "client_systems": { + "type": ["string", "null"] }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ], - "user_defined_primary_key": [ - [ - "id" - ] - ], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "tasks", - "json_schema": { - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "title": { - "type": "string" - }, - "description": { - "type": [ - "string", - "null" - ] - }, - "project": { - "type": "string" - }, - "status": { - "type": "string" - }, - "dependency": { - "type": [ - "string", - "null" - ] - }, - "start_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "due_datetime": { - "type": "string", - "format": "date-time" - }, - "started_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "finished_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "started_overridden_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "finished_overridden_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "estimated_duration": { - "type": [ - "integer", - "null" - ] - }, - "milestone": { - "type": "string" - }, - "created_by": { - "type": [ - "string", - "null" - ] - }, - "assigned_to": { - "type": [ - "string", - "null" - ] - }, - "created_from": { - "type": [ - "string", - "null" - ] - }, - "risk_profiles": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "risk_level": { - "type": "string" - }, - "formula": { - "type": "string" - }, - "over_run": { - "type": "integer" - }, - "task_variance": { - "type": "integer" - }, - "cool_down": { - "type": "integer" - }, - "reason": { - "type": "integer" - }, - "duration": { - "type": "integer" - }, - "estimated_duration": { - "type": "integer" - } - } - } - }, - "time_entry_list": { - "type": "string" - }, - "attachment_list": { - "type": "string" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } - } + "companies": { + "type": "array", + "items": { + "type": "string" + } + }, + "completed_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "cost": { + "type": ["integer", "null"] + }, + "created": { + "type": "string", + "format": "date-time" + }, + "created_from": { + "type": ["string", "null"] + }, + "created_from_template": { + "type": ["string", "null"] + }, + "creator": { + "type": ["string", "null"] + }, + "deadline_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "estimated_duration": { + "type": ["integer", "null"] + }, + "id": { + "type": "integer" + }, + "implementation_budget": { + "type": "string" + }, + "milestone_list": { + "type": "string" + }, + "modified": { + "type": ["string", "null"], + "format": "date-time" + }, + "phase": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "name": { + "type": ["string", "null"] + }, + "order": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" } + } }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ], - "user_defined_primary_key": [ - [ - "id" - ] - ], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "task_attachments", - "json_schema": { - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "task": { - "type": "string" - }, - "url": { - "type": "string" - }, - "type": { - "type": "string" - }, - "label": { - "type": [ - "string", - "null" - ] - }, - "deliverable": { - "type": "boolean" - }, - "requires_approval": { - "type": "boolean" - }, - "approved": { - "type": [ - "boolean", - "null" - ] - }, - "revision_task": { - "type": [ - "string", - "null" - ] - }, - "original_filename": { - "type": [ - "string", - "null" - ] - }, - "created_by": { - "type": [ - "string", - "null" - ] - }, - "created": { - "type": "string" - }, - "modified": { - "type": "string" - } - } + "risk_profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "risk_score": { + "type": "number" + }, + "level": { + "type": "string" + }, + "variance": { + "type": "integer" + }, + "formula": { + "type": "string" + }, + "projected_golive_datetime": { + "type": ["string", "null"], + "format": "date-time" + } } + } + }, + "start_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "started_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "status": { + "type": "string" + }, + "task_list": { + "type": "string" + }, + "time_entry_list": { + "type": "string" + }, + "title": { + "type": "string" + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "user_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "phases", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "name": { + "type": ["string", "null"] + }, + "order": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "user_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "project_attachments", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "project": { + "type": "string" + }, + "url": { + "type": "string" + }, + "label": { + "type": ["string", "null"] + }, + "created_by": { + "type": "string" + }, + "type": { + "type": "string" + }, + "is_sow": { + "type": "boolean" + }, + "original_filename": { + "type": ["string", "null"] + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "datetime" + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "user_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "tasks", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ], - "user_defined_primary_key": [ - [ - "id" - ] - ], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "templates", - "json_schema": { - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "title": { - "type": "string" - }, - "status": { - "type": "string" - }, - "cost": { - "type": [ - "string", - "null" - ] - }, - "annual_contract_value": { - "type": [ - "string", - "null" - ] - }, - "implementation_budget": { - "type": [ - "string", - "null" - ] - }, - "estimated_duration": { - "type": [ - "integer", - "null" - ] - }, - "created_from_template": { - "type": [ - "string", - "null" - ] - }, - "created_from": { - "type": [ - "string", - "null" - ] - }, - "start_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "started_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "deadline_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "completed_datetime": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "client_systems": { - "type": [ - "string", - "null" - ] - }, - "phase": { - "type": [ - "object", - "null" - ], - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "order": { - "type": "integer" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } - } - }, - "creator": { - "type": [ - "string", - "null" - ] - }, - "task_list": { - "type": "string" - } - } + "title": { + "type": "string" + }, + "description": { + "type": ["string", "null"] + }, + "project": { + "type": "string" + }, + "status": { + "type": "string" + }, + "dependency": { + "type": ["string", "null"] + }, + "start_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "due_datetime": { + "type": "string", + "format": "date-time" + }, + "started_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "finished_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "started_overridden_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "finished_overridden_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "estimated_duration": { + "type": ["integer", "null"] + }, + "milestone": { + "type": "string" + }, + "created_by": { + "type": ["string", "null"] + }, + "assigned_to": { + "type": ["string", "null"] + }, + "created_from": { + "type": ["string", "null"] + }, + "risk_profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "risk_level": { + "type": "string" + }, + "formula": { + "type": "string" + }, + "over_run": { + "type": "integer" + }, + "task_variance": { + "type": "integer" + }, + "cool_down": { + "type": "integer" + }, + "reason": { + "type": "integer" + }, + "duration": { + "type": "integer" + }, + "estimated_duration": { + "type": "integer" + } } + } + }, + "time_entry_list": { + "type": "string" + }, + "attachment_list": { + "type": "string" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "user_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "task_attachments", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "task": { + "type": "string" + }, + "url": { + "type": "string" + }, + "type": { + "type": "string" + }, + "label": { + "type": ["string", "null"] + }, + "deliverable": { + "type": "boolean" + }, + "requires_approval": { + "type": "boolean" + }, + "approved": { + "type": ["boolean", "null"] + }, + "revision_task": { + "type": ["string", "null"] + }, + "original_filename": { + "type": ["string", "null"] + }, + "created_by": { + "type": ["string", "null"] + }, + "created": { + "type": "string" + }, + "modified": { + "type": "string" + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "user_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "templates", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "title": { + "type": "string" + }, + "status": { + "type": "string" }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ], - "user_defined_primary_key": [ - [ - "id" - ] - ], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "time_entries", - "json_schema": { - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "user": { - "type": "string" - }, - "created_by": { - "type": "string" - }, - "project": { - "type": "string" - }, - "task": { - "type": [ - "string", - "null" - ] - }, - "rate": { - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "hourly_rate": { - "type": "string" - } - } - }, - "started_at": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "ended_at": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "reference_date": { - "type": "string", - "format": "date-time" - }, - "billable": { - "type": "boolean" - }, - "calculated_duration": { - "type": [ - "integer", - "null" - ] - }, - "inputted_duration": { - "type": [ - "integer", - "null" - ] - }, - "notes": { - "type": "string" - } - } + "cost": { + "type": ["string", "null"] + }, + "annual_contract_value": { + "type": ["string", "null"] + }, + "implementation_budget": { + "type": ["string", "null"] + }, + "estimated_duration": { + "type": ["integer", "null"] + }, + "created_from_template": { + "type": ["string", "null"] + }, + "created_from": { + "type": ["string", "null"] + }, + "start_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "started_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "deadline_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "completed_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "client_systems": { + "type": ["string", "null"] + }, + "phase": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "order": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" } + } + }, + "creator": { + "type": ["string", "null"] + }, + "task_list": { + "type": "string" + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "user_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "time_entries", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "user": { + "type": "string" + }, + "created_by": { + "type": "string" + }, + "project": { + "type": "string" + }, + "task": { + "type": ["string", "null"] }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ], - "user_defined_primary_key": [ - [ - "id" - ] - ], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "users", - "json_schema": { - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "first_name": { - "type": "string" - }, - "last_name": { - "type": "string" - }, - "email": { - "type": "string" - }, - "account_type": { - "type": "string" - }, - "job_title": { - "type": "string" - }, - "company": { - "type": "string" - }, - "avatar_url": { - "type": [ - "string", - "null" - ] - }, - "created_by": { - "type": [ - "string", - "null" - ] - }, - "signed_up_at": { - "type": [ - "string", - "null" - ], - "format": "date-time" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } - } + "rate": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "hourly_rate": { + "type": "string" } + } + }, + "started_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "ended_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "reference_date": { + "type": "string", + "format": "date-time" + }, + "billable": { + "type": "boolean" + }, + "calculated_duration": { + "type": ["integer", "null"] + }, + "inputted_duration": { + "type": ["integer", "null"] + }, + "notes": { + "type": "string" + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "user_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": { + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "account_type": { + "type": "string" + }, + "job_title": { + "type": "string" + }, + "company": { + "type": "string" + }, + "avatar_url": { + "type": ["string", "null"] + }, + "created_by": { + "type": ["string", "null"] + }, + "signed_up_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "created": { + "type": "string", + "format": "date-time" }, - "supported_sync_modes": [ - "full_refresh" - ], - "source_defined_primary_key": [ - [ - "id" - ] - ], - "user_defined_primary_key": [ - [ - "id" - ] - ], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "modified": { + "type": "string", + "format": "date-time" + } + } } - ] -} \ No newline at end of file + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "user_defined_primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-hellobaton/sample_files/sample_config.json b/airbyte-integrations/connectors/source-hellobaton/sample_files/sample_config.json index 312a9dfadae27..cc58f6f0f835c 100644 --- a/airbyte-integrations/connectors/source-hellobaton/sample_files/sample_config.json +++ b/airbyte-integrations/connectors/source-hellobaton/sample_files/sample_config.json @@ -1,4 +1,4 @@ { - "company": "google", - "api_key": "" + "company": "google", + "api_key": "" } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json index c21d941a29be1..cc4a631b4d81f 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json @@ -1,48 +1,48 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "type": { - "type": ["string", "null"] - }, - "group": { - "type": "string" - }, - "parent": { - "type": ["string", "null"] - }, - "child": { - "type": ["string", "null"] - }, - "actor": { - "type": ["string", "null"] - }, - "project": { - "type": ["string", "null"] - }, - "parent_type": { - "type": ["string", "null"] - }, - "child_type": { - "type": ["string", "null"] - }, - "meta": { - "type": ["object", "null"] - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "type": { + "type": ["string", "null"] + }, + "group": { + "type": "string" + }, + "parent": { + "type": ["string", "null"] + }, + "child": { + "type": ["string", "null"] + }, + "actor": { + "type": ["string", "null"] + }, + "project": { + "type": ["string", "null"] + }, + "parent_type": { + "type": ["string", "null"] + }, + "child_type": { + "type": ["string", "null"] + }, + "meta": { + "type": ["object", "null"] + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json index 11efb5fbbec6c..ebbb25699ee35 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json @@ -1,33 +1,32 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, - "id": { - "type": "integer" - }, + "_self": { + "type": "string" + }, - "_self": { - "type": "string" - }, + "name": { + "type": "string" + }, - "name": { - "type": "string" - }, + "type": { + "type": "string" + }, - "type": { - "type": "string" - }, + "created": { + "type": "string", + "format": "date-time" + }, - "created": { - "type": "string", - "format": "date-time" - }, - - "modified": { - "type": "string", - "format": "date-time" - } + "modified": { + "type": "string", + "format": "date-time" } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json index 1892900ff5f39..16e0f0060b470 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json @@ -1,95 +1,95 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": ["string", "null"] + }, + "project": { + "type": "string" + }, + "task_list": { + "type": "string" + }, + "phase": { + "type": ["object", "null"], + "properties": { "id": { - "type": "integer" + "type": "integer" }, "_self": { - "type": "string" + "type": "string" }, - "title": { - "type": "string" + "name": { + "type": "string" }, - "description": { - "type": ["string", "null"] + "created": { + "type": "string", + "format": "date-time" }, - "project": { + "modified": { + "type": "string", + "format": "date-time" + } + } + }, + "deadline_fixed": { + "type": "boolean" + }, + "deadline_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "risk_profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "risk_level": { "type": "string" - }, - "task_list": { + }, + "formula": { "type": "string" - }, - "phase": { - "type": ["object", "null"], - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "name": { - "type": "string" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } - } - }, - "deadline_fixed": { - "type": "boolean" - }, - "deadline_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "risk_profiles": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "risk_level": { - "type": "string" - }, - "formula": { - "type": "string" - }, - "over_run": { - "type": "integer" - } - } - } - }, - "start_datetime": { - "type": "string" - }, - "finish_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "created_from": { - "type": ["string", "null"] - }, - "duration": { + }, + "over_run": { "type": "integer" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" + } } + } + }, + "start_datetime": { + "type": "string" + }, + "finish_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "created_from": { + "type": ["string", "null"] + }, + "duration": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json index c1d915ea88c37..476437ea6774b 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json @@ -1,27 +1,27 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "name": { - "type": ["string", "null"] - }, - "order": { - "type": "integer" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "name": { + "type": ["string", "null"] + }, + "order": { + "type": "integer" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json index 256dab1f8ee38..7d90c8628a27d 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json @@ -1,42 +1,42 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "project": { - "type": "string" - }, - "url": { - "type": "string" - }, - "label": { - "type": ["string", "null"] - }, - "created_by": { - "type": "string" - }, - "type": { - "type": "string" - }, - "is_sow": { - "type": "boolean" - }, - "original_filename": { - "type": ["string", "null"] - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "datetime" - } + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "project": { + "type": "string" + }, + "url": { + "type": "string" + }, + "label": { + "type": ["string", "null"] + }, + "created_by": { + "type": "string" + }, + "type": { + "type": "string" + }, + "is_sow": { + "type": "boolean" + }, + "original_filename": { + "type": ["string", "null"] + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "datetime" } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json index 46add2d1362ab..e52493d79752b 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json @@ -1,140 +1,137 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - - "_self": { - "type": "string" - }, - "annual_contract_value": { - "type": "string" - }, - "attachment_list": { - "type": "string" - }, - "client_systems": { - "type": ["string", "null"] + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "_self": { + "type": "string" + }, + "annual_contract_value": { + "type": "string" + }, + "attachment_list": { + "type": "string" + }, + "client_systems": { + "type": ["string", "null"] + }, + "companies": { + "type": "array", + "items": { + "type": "string" + } + }, + "completed_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "cost": { + "type": ["integer", "null"] + }, + "created": { + "type": "string", + "format": "date-time" + }, + "created_from": { + "type": ["string", "null"] + }, + "created_from_template": { + "type": ["string", "null"] + }, + "creator": { + "type": ["string", "null"] + }, + "deadline_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "estimated_duration": { + "type": ["integer", "null"] + }, + "id": { + "type": "integer" + }, + "implementation_budget": { + "type": "string" + }, + "milestone_list": { + "type": "string" + }, + "modified": { + "type": ["string", "null"], + "format": "date-time" + }, + "phase": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "integer" }, - "companies": { - "type": "array", - "items": { - "type": "string" - } + "_self": { + "type": "string" }, - "completed_datetime": { - "type": ["string", "null"], - "format": "date-time" + "name": { + "type": ["string", "null"] }, - "cost": { - "type": ["integer", "null"] + "order": { + "type": "integer" }, "created": { - "type": "string", - "format": "date-time" - }, - "created_from": { - "type": ["string", "null"] - }, - "created_from_template": { - "type": ["string", "null"] + "type": "string", + "format": "date-time" }, - "creator": { - "type": ["string", "null"] - }, - "deadline_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "estimated_duration": { - "type": ["integer", "null"] - }, - "id": { + "modified": { + "type": "string", + "format": "date-time" + } + } + }, + "risk_profiles": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": "integer" - }, - "implementation_budget": { + }, + "risk_score": { + "type": "number" + }, + "level": { "type": "string" - }, - "milestone_list": { + }, + "variance": { + "type": "integer" + }, + "formula": { "type": "string" - }, - "modified": { + }, + "projected_golive_datetime": { "type": ["string", "null"], "format": "date-time" - }, - "phase": { - "type": ["object", "null"], - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "name": { - "type": ["string", "null"] - }, - "order": { - "type": "integer" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } - } - }, - "risk_profiles": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "risk_score": { - "type": "number" - }, - "level": { - "type": "string" - }, - "variance": { - "type": "integer" - }, - "formula": { - "type": "string" - }, - "projected_golive_datetime": { - "type": ["string", "null"], - "format": "date-time" - } - } - } - }, - "start_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "started_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "status": { - "type": "string" - }, - "task_list": { - "type": "string" - }, - "time_entry_list": { - "type": "string" - }, - "title": { - "type": "string" + } } - + } + }, + "start_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "started_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "status": { + "type": "string" + }, + "task_list": { + "type": "string" + }, + "time_entry_list": { + "type": "string" + }, + "title": { + "type": "string" } - -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json index 0ad60ebfa2d59..90da99ec9d51f 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json @@ -1,49 +1,49 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "task": { - "type": "string" - }, - "url": { - "type": "string" - }, - "type": { - "type": "string" - }, - "label": { - "type": ["string", "null"] - }, - "deliverable": { - "type": "boolean" - }, - "requires_approval": { - "type": "boolean" - }, - "approved": { - "type": ["boolean", "null"] - }, - "revision_task": { - "type": ["string", "null"] - }, - "original_filename": { - "type": ["string", "null"] - }, - "created_by": { - "type": ["string", "null"] - }, - "created": { - "type": "string" - }, - "modified": { - "type": "string" - } + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "task": { + "type": "string" + }, + "url": { + "type": "string" + }, + "type": { + "type": "string" + }, + "label": { + "type": ["string", "null"] + }, + "deliverable": { + "type": "boolean" + }, + "requires_approval": { + "type": "boolean" + }, + "approved": { + "type": ["boolean", "null"] + }, + "revision_task": { + "type": ["string", "null"] + }, + "original_filename": { + "type": ["string", "null"] + }, + "created_by": { + "type": ["string", "null"] + }, + "created": { + "type": "string" + }, + "modified": { + "type": "string" } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json index c6122b86eb0df..980c82d5f5ced 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json @@ -113,4 +113,4 @@ "format": "date-time" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json index d5335f9292908..e2d02743d883a 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json @@ -1,96 +1,84 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "title": { + "type": "string" + }, + "status": { + "type": "string" + }, + "cost": { + "type": ["string", "null"] + }, + "annual_contract_value": { + "type": ["string", "null"] + }, + "implementation_budget": { + "type": ["string", "null"] + }, + "estimated_duration": { + "type": ["integer", "null"] + }, + "created_from_template": { + "type": ["string", "null"] + }, + "created_from": { + "type": ["string", "null"] + }, + "start_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "started_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "deadline_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "completed_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "client_systems": { + "type": ["string", "null"] + }, + "phase": { + "type": ["object", "null"], + "properties": { "id": { - "type": "integer" + "type": "integer" }, "_self": { - "type": "string" + "type": "string" }, - "title": { - "type": "string" + "order": { + "type": "integer" }, - "status": { - "type": "string" + "created": { + "type": "string", + "format": "date-time" }, - "cost": { - "type": [ - "string", - "null" - ] - }, - "annual_contract_value": { - "type": [ - "string", - "null" - ] - }, - "implementation_budget": { - "type": [ - "string", - "null" - ] - }, - "estimated_duration": { - "type": [ - "integer", - "null" - ] - }, - "created_from_template": { - "type": ["string", "null"] - }, - "created_from": { - "type": ["string", "null"] - }, - "start_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "started_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "deadline_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "completed_datetime": { - "type": ["string", "null"], - "format": "date-time" - }, - "client_systems": { - "type": ["string", "null"] - }, - "phase": { - "type": ["object", "null"], - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "order": { - "type": "integer" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } - } - }, - "creator": { - "type": ["string", "null"] - }, - "task_list": { - "type": "string" + "modified": { + "type": "string", + "format": "date-time" } + } + }, + "creator": { + "type": ["string", "null"] + }, + "task_list": { + "type": "string" } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json index 76af78e44d0d1..7ea2f5553be25 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json @@ -1,60 +1,60 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "user": { + "type": "string" + }, + "created_by": { + "type": "string" + }, + "project": { + "type": "string" + }, + "task": { + "type": ["string", "null"] + }, + "rate": { + "type": "object", + "properties": { "id": { - "type": "integer" + "type": "integer" }, - "_self": { - "type": "string" - }, - "user": { - "type": "string" - }, - "created_by": { - "type": "string" - }, - "project": { - "type": "string" - }, - "task": { - "type": ["string", "null"] - }, - "rate": { - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "hourly_rate": { - "type": "string" - } - } - }, - "started_at": { - "type": ["string", "null"], - "format": "date-time" - }, - "ended_at": { - "type": ["string", "null"], - "format": "date-time" - }, - "reference_date": { - "type": "string", - "format": "date-time" - }, - "billable": { - "type": "boolean" - }, - "calculated_duration": { - "type": ["integer", "null"] - }, - "inputted_duration": { - "type": ["integer", "null"] - }, - "notes": { - "type": "string" + "hourly_rate": { + "type": "string" } + } + }, + "started_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "ended_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "reference_date": { + "type": "string", + "format": "date-time" + }, + "billable": { + "type": "boolean" + }, + "calculated_duration": { + "type": ["integer", "null"] + }, + "inputted_duration": { + "type": ["integer", "null"] + }, + "notes": { + "type": "string" } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json index 76faf58fedfc1..3e6d3d73a6037 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json @@ -1,49 +1,49 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": "integer" - }, - "_self": { - "type": "string" - }, - "first_name": { - "type": "string" - }, - "last_name": { - "type": "string" - }, - "email": { - "type": "string" - }, - "account_type": { - "type": "string" - }, - "job_title": { - "type": "string" - }, - "company": { - "type": "string" - }, - "avatar_url": { - "type": ["string", "null"] - }, - "created_by": { - "type": ["string", "null"] - }, - "signed_up_at": { - "type": ["string", "null"], - "format": "date-time" - }, - "created": { - "type": "string", - "format": "date-time" - }, - "modified": { - "type": "string", - "format": "date-time" - } + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "_self": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "account_type": { + "type": "string" + }, + "job_title": { + "type": "string" + }, + "company": { + "type": "string" + }, + "avatar_url": { + "type": ["string", "null"] + }, + "created_by": { + "type": ["string", "null"] + }, + "signed_up_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "created": { + "type": "string", + "format": "date-time" + }, + "modified": { + "type": "string", + "format": "date-time" } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/spec.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/spec.json index 3f286953ec5ec..50793f64fae2e 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/spec.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/spec.json @@ -1,22 +1,22 @@ { - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Hellobaton Spec", - "type": "object", - "required": ["api_key", "company"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "authentication key required to access the api endpoints", - "airbyte_secret": true - }, - "company": { - "type": "string", - "description": "Company name that generates your base api url", - "examples": ["google", "facebook", "microsoft"] - } - } + "documentationUrl": "https://docsurl.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Hellobaton Spec", + "type": "object", + "required": ["api_key", "company"], + "additionalProperties": false, + "properties": { + "api_key": { + "type": "string", + "description": "authentication key required to access the api endpoints", + "airbyte_secret": true + }, + "company": { + "type": "string", + "description": "Company name that generates your base api url", + "examples": ["google", "facebook", "microsoft"] + } } -} \ No newline at end of file + } +} From fc1ef65cf328390011524cda58aa8b4b43334554 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Thu, 13 Jan 2022 23:06:42 -0300 Subject: [PATCH 14/17] bump version in config files --- .../492b56d1-937c-462e-8076-21ad2031e784.json | 8 ++++++++ .../init/src/main/resources/seed/source_definitions.yaml | 6 ++++++ 2 files changed, 14 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/492b56d1-937c-462e-8076-21ad2031e784.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/492b56d1-937c-462e-8076-21ad2031e784.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/492b56d1-937c-462e-8076-21ad2031e784.json new file mode 100644 index 0000000000000..e6beb76ecda3a --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/492b56d1-937c-462e-8076-21ad2031e784.json @@ -0,0 +1,8 @@ +{ + "sourceDefinitionId": "492b56d1-937c-462e-8076-21ad2031e784", + "name": "Hellobaton", + "dockerRepository": "airbyte/source-hellobaton", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/hellobaton" + } + \ No newline at end of file diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 17551d21c8e43..9fa9564705188 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -284,6 +284,12 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/harvest icon: harvest.svg sourceType: api +- name: Hellobaton + sourceDefinitionId: 492b56d1-937c-462e-8076-21ad2031e784 + dockerRepository: airbyte/source-hellobaton + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/hellobaton + sourceType: api - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot From d62f48804c95f08b249d1b76d6b4ab763c53a5ca Mon Sep 17 00:00:00 2001 From: Daniel Luftspring Date: Thu, 13 Jan 2022 21:24:27 -0500 Subject: [PATCH 15/17] Docs page for hellobaton --- docs/integrations/sources/hellobaton.md | 50 +++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/integrations/sources/hellobaton.md diff --git a/docs/integrations/sources/hellobaton.md b/docs/integrations/sources/hellobaton.md new file mode 100644 index 0000000000000..16d8d401d884f --- /dev/null +++ b/docs/integrations/sources/hellobaton.md @@ -0,0 +1,50 @@ +# Baton + +## Sync overview + +This source can sync data from the [baton API](https://app.hellobaton.com/api/redoc/). At present this connector only supports full refresh syncs meaning that each time you use the connector it will sync all available records from scratch. Please use cautiously if you expect your API to have a lot of records. + +## This Source Supports the Following Streams + +* activity +* companies +* milestones +* phases +* project_attachments +* projects +* task_attachemnts +* tasks +* templates +* time_entries +* users + +Baton adds new streams fairly regularly please submit an issue or PR if this project doesn't support required streams for your use case. + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--- | :--- | :--- | +| `string` | `string` | | +| `integer` | `integer` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | +| Namespaces | No | | + +### Performance considerations + +The connector is rate limited at 1000 requests per minute per api key. If you find yourself receiving errors contact your customer success manager and request a rate limit increase. + +## Getting started + +### Requirements + +* Baton account +* Baton api key \ No newline at end of file From 7523844d50bd76466c2a8d0bdb0b843cb19d1184 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Fri, 14 Jan 2022 00:21:05 -0300 Subject: [PATCH 16/17] add doc --- .../492b56d1-937c-462e-8076-21ad2031e784.json | 13 +++--- .../AzureBlobStorageDestinationConfig.java | 4 +- .../csv/AzureBlobStorageCsvWriter.java | 1 - .../jsonl/AzureBlobStorageJsonlWriter.java | 1 - .../snowflake/SnowflakeDestinationTest.java | 43 +++++++++---------- 5 files changed, 29 insertions(+), 33 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/492b56d1-937c-462e-8076-21ad2031e784.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/492b56d1-937c-462e-8076-21ad2031e784.json index e6beb76ecda3a..6f5f3febef56b 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/492b56d1-937c-462e-8076-21ad2031e784.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/492b56d1-937c-462e-8076-21ad2031e784.json @@ -1,8 +1,7 @@ { - "sourceDefinitionId": "492b56d1-937c-462e-8076-21ad2031e784", - "name": "Hellobaton", - "dockerRepository": "airbyte/source-hellobaton", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://docs.airbyte.io/integrations/sources/hellobaton" - } - \ No newline at end of file + "sourceDefinitionId": "492b56d1-937c-462e-8076-21ad2031e784", + "name": "Hellobaton", + "dockerRepository": "airbyte/source-hellobaton", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/hellobaton" +} diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestinationConfig.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestinationConfig.java index 8d575214b678c..1cebcd4b600cd 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestinationConfig.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestinationConfig.java @@ -64,11 +64,11 @@ public static AzureBlobStorageDestinationConfig getAzureBlobStorageConfig(final final JsonNode endpointFromConfig = config .get("azure_blob_storage_endpoint_domain_name"); final JsonNode containerName = config.get("azure_blob_storage_container_name"); - final int outputStreamBufferSizeFromConfig = + final int outputStreamBufferSizeFromConfig = config.get("azure_blob_storage_output_buffer_size") != null ? config.get("azure_blob_storage_output_buffer_size").asInt(DEFAULT_STORAGE_OUTPUT_BUFFER_SIZE) : DEFAULT_STORAGE_OUTPUT_BUFFER_SIZE; - + final JsonNode blobName = config.get("azure_blob_storage_blob_name"); // streamId final String endpointComputed = String.format(Locale.ROOT, DEFAULT_STORAGE_ENDPOINT_FORMAT, diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/csv/AzureBlobStorageCsvWriter.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/csv/AzureBlobStorageCsvWriter.java index 23e31bbf4d9ce..f31aaab64fa02 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/csv/AzureBlobStorageCsvWriter.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/csv/AzureBlobStorageCsvWriter.java @@ -5,7 +5,6 @@ package io.airbyte.integrations.destination.azure_blob_storage.csv; import com.azure.storage.blob.specialized.AppendBlobClient; -import com.azure.storage.blob.specialized.BlobOutputStream; import io.airbyte.integrations.destination.azure_blob_storage.AzureBlobStorageDestinationConfig; import io.airbyte.integrations.destination.azure_blob_storage.writer.AzureBlobStorageWriter; import io.airbyte.integrations.destination.azure_blob_storage.writer.BaseAzureBlobStorageWriter; diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/jsonl/AzureBlobStorageJsonlWriter.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/jsonl/AzureBlobStorageJsonlWriter.java index 6a0406be7a7e2..77ed63b7dee63 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/jsonl/AzureBlobStorageJsonlWriter.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/jsonl/AzureBlobStorageJsonlWriter.java @@ -5,7 +5,6 @@ package io.airbyte.integrations.destination.azure_blob_storage.jsonl; import com.azure.storage.blob.specialized.AppendBlobClient; -import com.azure.storage.blob.specialized.BlobOutputStream; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.node.ObjectNode; diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationTest.java index 021fa9ed88feb..047fe909a6c85 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationTest.java @@ -24,15 +24,14 @@ import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitive; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - import java.nio.file.Path; import java.sql.SQLException; import java.time.Instant; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; public class SnowflakeDestinationTest { @@ -84,40 +83,40 @@ public void testCleanupStageOnFailure() throws Exception { final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of("secrets/insert_config.json"))); AirbyteMessageConsumer airbyteMessageConsumer = new SnowflakeInternalStagingConsumerFactory() - .create(Destination::defaultOutputRecordCollector, mockDb, + .create(Destination::defaultOutputRecordCollector, mockDb, sqlOperations, new SnowflakeSQLNameTransformer(), config, getCatalog()); - doThrow(SQLException.class).when(sqlOperations).copyIntoTmpTableFromStage(any(),anyString(),anyString(),anyString()); + doThrow(SQLException.class).when(sqlOperations).copyIntoTmpTableFromStage(any(), anyString(), anyString(), anyString()); airbyteMessageConsumer.start(); for (AirbyteMessage m : testMessages) { - airbyteMessageConsumer.accept(m); + airbyteMessageConsumer.accept(m); } assertThrows(RuntimeException.class, airbyteMessageConsumer::close); - verify(sqlOperations, times(1)).cleanUpStage(any(),anyString()); + verify(sqlOperations, times(1)).cleanUpStage(any(), anyString()); } private List generateTestMessages() { return IntStream.range(0, 3) - .boxed() - .map(i -> new AirbyteMessage() - .withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage() - .withStream("test") - .withNamespace("test_staging") - .withEmittedAt(Instant.now().toEpochMilli()) - .withData(Jsons.jsonNode(ImmutableMap.of("id", i, "name", "human " + i))))) - .collect(Collectors.toList()); + .boxed() + .map(i -> new AirbyteMessage() + .withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream("test") + .withNamespace("test_staging") + .withEmittedAt(Instant.now().toEpochMilli()) + .withData(Jsons.jsonNode(ImmutableMap.of("id", i, "name", "human " + i))))) + .collect(Collectors.toList()); } ConfiguredAirbyteCatalog getCatalog() { return new ConfiguredAirbyteCatalog().withStreams(List.of( - CatalogHelpers.createConfiguredAirbyteStream( - "test", - "test_staging", - Field.of("id", JsonSchemaPrimitive.NUMBER), - Field.of("name", JsonSchemaPrimitive.STRING)) - .withDestinationSyncMode(DestinationSyncMode.OVERWRITE))); + CatalogHelpers.createConfiguredAirbyteStream( + "test", + "test_staging", + Field.of("id", JsonSchemaPrimitive.NUMBER), + Field.of("name", JsonSchemaPrimitive.STRING)) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE))); } } From d7baaa86d8d30659b14af8f6378f487de73e5186 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Fri, 14 Jan 2022 14:51:26 -0300 Subject: [PATCH 17/17] run format --- .../src/main/resources/seed/source_specs.yaml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index ed9459b0c5bbc..4909346bbea47 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2846,6 +2846,32 @@ path_in_connector_config: - "credentials" - "client_secret" +- dockerImage: "airbyte/source-hellobaton:0.1.0" + spec: + documentationUrl: "https://docsurl.com" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Hellobaton Spec" + type: "object" + required: + - "api_key" + - "company" + additionalProperties: false + properties: + api_key: + type: "string" + description: "authentication key required to access the api endpoints" + airbyte_secret: true + company: + type: "string" + description: "Company name that generates your base api url" + examples: + - "google" + - "facebook" + - "microsoft" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-hubspot:0.1.32" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot"