From d1a58a1cfe333638096986b80c0c540bf7ebbf15 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 21 Dec 2021 12:22:36 +0100 Subject: [PATCH 01/22] =?UTF-8?q?=F0=9F=90=99=20Bootstrap=20octavia=20proj?= =?UTF-8?q?ect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + octavia-cli/.coveragerc | 3 ++ octavia-cli/.dockerignore | 3 ++ octavia-cli/.gitignore | 2 + octavia-cli/.python-version | 1 + octavia-cli/CHANGELOG.md | 0 octavia-cli/Dockerfile | 17 +++++++ octavia-cli/LICENSE | 21 +++++++++ octavia-cli/README.md | 55 +++++++++++++++++++++++ octavia-cli/build.gradle | 9 ++++ octavia-cli/octavia_cli/__init__.py | 0 octavia-cli/octavia_cli/entrypoint.py | 42 +++++++++++++++++ octavia-cli/pytest.ini | 5 +++ octavia-cli/setup.py | 53 ++++++++++++++++++++++ octavia-cli/unit_tests/test_entrypoint.py | 25 +++++++++++ settings.gradle | 1 + 16 files changed, 238 insertions(+) create mode 100644 octavia-cli/.coveragerc create mode 100644 octavia-cli/.dockerignore create mode 100644 octavia-cli/.gitignore create mode 100644 octavia-cli/.python-version create mode 100644 octavia-cli/CHANGELOG.md create mode 100644 octavia-cli/Dockerfile create mode 100644 octavia-cli/LICENSE create mode 100644 octavia-cli/README.md create mode 100644 octavia-cli/build.gradle create mode 100644 octavia-cli/octavia_cli/__init__.py create mode 100644 octavia-cli/octavia_cli/entrypoint.py create mode 100644 octavia-cli/pytest.ini create mode 100644 octavia-cli/setup.py create mode 100644 octavia-cli/unit_tests/test_entrypoint.py diff --git a/.gitignore b/.gitignore index b114af0e0fb35..37f990d595681 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .gradle .idea +.vscode *.iml *.swp build diff --git a/octavia-cli/.coveragerc b/octavia-cli/.coveragerc new file mode 100644 index 0000000000000..034c0c0c28f5e --- /dev/null +++ b/octavia-cli/.coveragerc @@ -0,0 +1,3 @@ +[report] +# show lines missing coverage +show_missing = true diff --git a/octavia-cli/.dockerignore b/octavia-cli/.dockerignore new file mode 100644 index 0000000000000..91d02a484a111 --- /dev/null +++ b/octavia-cli/.dockerignore @@ -0,0 +1,3 @@ +build +.venv +octavia_cli.egg-info \ No newline at end of file diff --git a/octavia-cli/.gitignore b/octavia-cli/.gitignore new file mode 100644 index 0000000000000..4cfda0ee73016 --- /dev/null +++ b/octavia-cli/.gitignore @@ -0,0 +1,2 @@ +.coverage +.venv \ No newline at end of file diff --git a/octavia-cli/.python-version b/octavia-cli/.python-version new file mode 100644 index 0000000000000..30291cba2230c --- /dev/null +++ b/octavia-cli/.python-version @@ -0,0 +1 @@ +3.10.0 diff --git a/octavia-cli/CHANGELOG.md b/octavia-cli/CHANGELOG.md new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/octavia-cli/Dockerfile b/octavia-cli/Dockerfile new file mode 100644 index 0000000000000..dd831f75c4304 --- /dev/null +++ b/octavia-cli/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.10.0-slim as base + +RUN apt-get upgrade \ + && pip install --upgrade pip + +WORKDIR /home/octavia-cli +COPY . ./ + +RUN pip install --no-cache-dir . + +RUN useradd --create-home --shell /bin/bash octavia-cli +USER octavia-cli + +ENTRYPOINT ["octavia"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/octavia-cli diff --git a/octavia-cli/LICENSE b/octavia-cli/LICENSE new file mode 100644 index 0000000000000..0e208b4b49e34 --- /dev/null +++ b/octavia-cli/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Airbyte, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/octavia-cli/README.md b/octavia-cli/README.md new file mode 100644 index 0000000000000..a7a61d02a23a3 --- /dev/null +++ b/octavia-cli/README.md @@ -0,0 +1,55 @@ +# 🐙 Octavia CLI + +Octavia CLI is a tool to manage Airbyte configuration in YAML. +It has the following features: +* Scaffolding of a readable directory architecture that will host the YAML configs. +* Auto-generation of YAML config file that matches the resources' schemas. +* Manage Airbyte resources with YAML config files. +* Safe resources update through diff display and validation. +* Simple secret management to avoid versioning credentials. + +The project is under development: readers can refer to our [tech spec deck](https://docs.google.com/presentation/d/10RjkCzBiVhCivnjSh63icYI7wG6S0N0ZIErEIsmXTqM/edit?usp=sharing) for an introduction to the tool. + +# Usage +We encourage users to use the CLI with docker to avoid the hassle of setting up a Python installation. +The project is under development: we have not yet published any docker image to our Docker registry. + +1. Build the image locally: +```bash +docker build -t octavia-cli:dev --rm . +``` +2. Run the CLI from docker: +```bash +docker run octavia-cli:dev +```` +3. Create an `octavia` alias in your `.bashrc` or `.zshrc`: +````bash +echo 'alias octavia="docker run octavia-cli:dev"' >> ~/.zshrc +source ~/.zshrc +octavia +```` + +# Current development status +Octavia is currently under development. +You can find a detailed and updated execution plan [here](https://docs.google.com/spreadsheets/d/1weB9nf0Zx3IR_QvpkxtjBAzyfGb7B0PWpsVt6iMB5Us/edit#gid=0). +We welcome community contributions! + +Summary of achievements: + +| Date | Milestone | +|------------|-------------------------------------| +| 2021-12-22 | Bootstrapping the project's code base | + +# Developing locally +1. Install Python 3.10.0. We suggest doing it through `pyenv` +2. Create a virtualenv: `python -m venv .venv` +3. Activate the virtualenv: `source .venv/bin/activate` +4. Install dev dependencies: `pip install -e .\[dev\]` +5. Install `pre-commit` hooks: `pre-commit install` +6. Run the test suite: `pytest --cov=octavia_cli unit_tests` +7. Iterate; please check the [Contributing](#contributing) for instructions on contributing. + +# Contributing +1. Please sign up to [Airbyte's Slack workspace](https://slack.airbyte.io/) and join the `#octavia-cli`. We'll sync up community efforts in this channel. +2. Read the [execution plan](https://docs.google.com/spreadsheets/d/1weB9nf0Zx3IR_QvpkxtjBAzyfGb7B0PWpsVt6iMB5Us/edit#gid=0) and find a task you'd like to work on. +3. Open a PR, make sure to test your code thoroughly. \ No newline at end of file diff --git a/octavia-cli/build.gradle b/octavia-cli/build.gradle new file mode 100644 index 0000000000000..5ebd064d689b0 --- /dev/null +++ b/octavia-cli/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' +} + +airbytePython { + moduleDirectory 'octavia_cli' +} + diff --git a/octavia-cli/octavia_cli/__init__.py b/octavia-cli/octavia_cli/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/octavia-cli/octavia_cli/entrypoint.py b/octavia-cli/octavia_cli/entrypoint.py new file mode 100644 index 0000000000000..3d82bc32f5ea3 --- /dev/null +++ b/octavia-cli/octavia_cli/entrypoint.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import click + + +@click.group() +@click.option("--airbyte-url", envvar="AIRBYTE_URL", default="http://localhost:8000", help="The URL of your Airbyte instance.") +def octavia(airbyte_url): + # TODO: check if the airbyte_url is reachable + click.secho(f"🐙 - Octavia is targetting your Airbyte instance running at {airbyte_url}") + + +@octavia.command(help="Scaffolds a local project directories.") +def init(): + raise click.ClickException("The init command is not yet implemented.") + + +@octavia.command(name="list", help="List existing resources on the Airbyte instance.") +def _list(): + raise click.ClickException("The init command is not yet implemented.") + + +@octavia.command(name="import", help="Import an existing resources from the Airbyte instance.") +def _import(): + raise click.ClickException("The init command is not yet implemented.") + + +@octavia.command(help="Generate a YAML configuration file to manage a resource.") +def create(): + raise click.ClickException("The init command is not yet implemented.") + + +@octavia.command(help="Create or update resources according to YAML configurations.") +def apply(): + raise click.ClickException("The init command is not yet implemented.") + + +@octavia.command(help="Delete resources") +def delete(): + raise click.ClickException("The init command is not yet implemented.") diff --git a/octavia-cli/pytest.ini b/octavia-cli/pytest.ini new file mode 100644 index 0000000000000..f5276a231cadd --- /dev/null +++ b/octavia-cli/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +log_cli = 1 +log_cli_level = INFO +log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) +log_cli_date_format=%Y-%m-%d %H:%M:%S diff --git a/octavia-cli/setup.py b/octavia-cli/setup.py new file mode 100644 index 0000000000000..279107d67e6da --- /dev/null +++ b/octavia-cli/setup.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import pathlib + +from setuptools import find_packages, setup + +# The directory containing this file +HERE = pathlib.Path(__file__).parent + +# The text of the README file +README = (HERE / "README.md").read_text() + +setup( + name="octavia-cli", + version="0.1.0", + description="A command line interface to manage Airbyte configurations", + long_description=README, + author="Airbyte", + author_email="contact@airbyte.io", + license="MIT", + url="https://github.com/airbytehq/airbyte", + classifiers=[ + # This information is used when browsing on PyPi. + # Dev Status + "Development Status :: 3 - Alpha", + # Project Audience + "Intended Audience :: Developers", + "Topic :: Scientific/Engineering", + "Topic :: Software Development :: Libraries :: Python Modules", + "License :: OSI Approved :: MIT License", + # Python Version Support + "Programming Language :: Python :: 3.10", + ], + keywords="airbyte cli command-line-interface configuration", + project_urls={ + "Documentation": "https://docs.airbyte.io/", + "Source": "https://github.com/airbytehq/airbyte", + "Tracker": "https://github.com/airbytehq/airbyte/issues", + }, + packages=find_packages(exclude=("tests", "docs")), + install_requires=["click~=8.0.3"], + python_requires=">=3.10.0", + extras_require={ + "dev": ["MyPy~=0.812", "pytest~=6.2.5", "pytest-cov", "pytest-mock", "requests-mock", "pre-commit"], + "sphinx-docs": [ + "Sphinx~=4.2", + "sphinx-rtd-theme~=1.0", + ], + }, + entry_points={"console_scripts": ["octavia=octavia_cli.entrypoint:octavia"]}, +) diff --git a/octavia-cli/unit_tests/test_entrypoint.py b/octavia-cli/unit_tests/test_entrypoint.py new file mode 100644 index 0000000000000..c8effc674c65f --- /dev/null +++ b/octavia-cli/unit_tests/test_entrypoint.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import pytest +from click.testing import CliRunner +from octavia_cli import entrypoint + + +def test_octavia(): + runner = CliRunner() + result = runner.invoke(entrypoint.octavia) + assert result.exit_code == 0 + assert result.output.startswith("Usage: octavia [OPTIONS] COMMAND [ARGS]...") + + +@pytest.mark.parametrize( + "command", + [entrypoint.init, entrypoint.apply, entrypoint.create, entrypoint.delete, entrypoint._list, entrypoint._import], +) +def test_not_implemented_commands(command): + runner = CliRunner() + result = runner.invoke(command) + assert result.exit_code == 1 + assert result.output.endswith("not yet implemented.\n") diff --git a/settings.gradle b/settings.gradle index 56da8a917448b..d974be91613c6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -67,6 +67,7 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD" include ':airbyte-temporal' include ':airbyte-tests' include ':airbyte-webapp' + include ':octavia-cli' } // connectors base From 03945ca50e3b691933b56b9de226057e33e8b744 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 22 Dec 2021 22:47:53 +0100 Subject: [PATCH 02/22] fix missing end lines --- octavia-cli/.dockerignore | 2 +- octavia-cli/.gitignore | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/octavia-cli/.dockerignore b/octavia-cli/.dockerignore index 91d02a484a111..89b498ce9934e 100644 --- a/octavia-cli/.dockerignore +++ b/octavia-cli/.dockerignore @@ -1,3 +1,3 @@ build .venv -octavia_cli.egg-info \ No newline at end of file +octavia_cli.egg-info diff --git a/octavia-cli/.gitignore b/octavia-cli/.gitignore index 4cfda0ee73016..5ffed900e0bce 100644 --- a/octavia-cli/.gitignore +++ b/octavia-cli/.gitignore @@ -1,2 +1,2 @@ .coverage -.venv \ No newline at end of file +.venv From ee4df772562feff2bc904b94851e7544149d5010 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 23 Dec 2021 20:08:25 +0100 Subject: [PATCH 03/22] upgrade pip --- buildSrc/src/main/groovy/airbyte-python.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index 36b37ab8575dc..23f154e8a9c18 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -69,7 +69,7 @@ class AirbytePythonPlugin implements Plugin { pip 'isort:5.6.4' pip 'pytest:6.1.2' pip 'pytest-cov:2.12.1' - pip 'pip:21.1.3' + pip 'pip:21.3.1' } project.task('isortFormat', type: PythonTask) { From 3bd5cadca89ce5e5755da6d23b262fcc3050bcdf Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 23 Dec 2021 20:11:51 +0100 Subject: [PATCH 04/22] set version to dev --- octavia-cli/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octavia-cli/Dockerfile b/octavia-cli/Dockerfile index dd831f75c4304..3c745e4ed8b32 100644 --- a/octavia-cli/Dockerfile +++ b/octavia-cli/Dockerfile @@ -13,5 +13,5 @@ USER octavia-cli ENTRYPOINT ["octavia"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=dev LABEL io.airbyte.name=airbyte/octavia-cli From 18216c5705397c62744073ba3ef1417e89d83104 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 23 Dec 2021 20:22:04 +0100 Subject: [PATCH 05/22] try python 3.7.9 to fix build failure in CI --- octavia-cli/.python-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octavia-cli/.python-version b/octavia-cli/.python-version index 30291cba2230c..c77a7de85cc88 100644 --- a/octavia-cli/.python-version +++ b/octavia-cli/.python-version @@ -1 +1 @@ -3.10.0 +3.7.9 From d5e39c7419bd677ef078584a9a814eb6fd630154 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 23 Dec 2021 20:23:03 +0100 Subject: [PATCH 06/22] revert pip upgrade --- buildSrc/src/main/groovy/airbyte-python.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index 23f154e8a9c18..36b37ab8575dc 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -69,7 +69,7 @@ class AirbytePythonPlugin implements Plugin { pip 'isort:5.6.4' pip 'pytest:6.1.2' pip 'pytest-cov:2.12.1' - pip 'pip:21.3.1' + pip 'pip:21.1.3' } project.task('isortFormat', type: PythonTask) { From dedb74a8a757423fdcedd80aa24f3ea689f746f9 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 23 Dec 2021 20:28:01 +0100 Subject: [PATCH 07/22] revert: try python 3.7.9 to fix build failure in CI --- octavia-cli/.python-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octavia-cli/.python-version b/octavia-cli/.python-version index c77a7de85cc88..e06d07afe14cc 100644 --- a/octavia-cli/.python-version +++ b/octavia-cli/.python-version @@ -1 +1 @@ -3.7.9 +3.10.0 \ No newline at end of file From e6e4cf570d106e66a387a312dee50341ac8f4787 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Thu, 30 Dec 2021 16:36:19 +0800 Subject: [PATCH 08/22] Does this work? --- .github/workflows/gradle.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 836b89f758a10..b95e914fd120f 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -216,6 +216,10 @@ jobs: with: node-version: "16.13.0" + - uses: actions/setup-python@v2 + with: + python-version: "3.7" + - name: Set up CI Gradle Properties run: | mkdir -p ~/.gradle/ From 3178a6c60ffae4732b64bd8e841699a4069bcc40 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 30 Dec 2021 09:54:56 +0100 Subject: [PATCH 09/22] install python 3.10 in platform-build --- .github/workflows/gradle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index b95e914fd120f..ee5d8c02db67b 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -218,7 +218,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.10" - name: Set up CI Gradle Properties run: | From 3219d2a680b1e081f80072c655d639d9c4135809 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 30 Dec 2021 10:12:38 +0100 Subject: [PATCH 10/22] downgrade to 3.8.12 --- .github/workflows/gradle.yml | 2 +- octavia-cli/.python-version | 2 +- octavia-cli/Dockerfile | 2 +- octavia-cli/setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index ee5d8c02db67b..6cb26f6ac3cbc 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -218,7 +218,7 @@ jobs: - uses: actions/setup-python@v2 with: - python-version: "3.10" + python-version: "3.8" - name: Set up CI Gradle Properties run: | diff --git a/octavia-cli/.python-version b/octavia-cli/.python-version index e06d07afe14cc..55e6aabe60527 100644 --- a/octavia-cli/.python-version +++ b/octavia-cli/.python-version @@ -1 +1 @@ -3.10.0 \ No newline at end of file +3.8.12 \ No newline at end of file diff --git a/octavia-cli/Dockerfile b/octavia-cli/Dockerfile index 3c745e4ed8b32..9313773b20d4b 100644 --- a/octavia-cli/Dockerfile +++ b/octavia-cli/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10.0-slim as base +FROM python:3.8.12-slim as base RUN apt-get upgrade \ && pip install --upgrade pip diff --git a/octavia-cli/setup.py b/octavia-cli/setup.py index 279107d67e6da..93534a6e851a6 100644 --- a/octavia-cli/setup.py +++ b/octavia-cli/setup.py @@ -41,7 +41,7 @@ }, packages=find_packages(exclude=("tests", "docs")), install_requires=["click~=8.0.3"], - python_requires=">=3.10.0", + python_requires=">=3.8.12", extras_require={ "dev": ["MyPy~=0.812", "pytest~=6.2.5", "pytest-cov", "pytest-mock", "requests-mock", "pre-commit"], "sphinx-docs": [ From fb7736f3937ed8974ecd492dd1f2dca20db408d4 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 30 Dec 2021 10:37:40 +0100 Subject: [PATCH 11/22] specific subbuild for octavia-cli --- .github/workflows/gradle.yml | 73 ++++++++++++++++++++++++++++++++++++ settings.gradle | 11 +++--- 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 6cb26f6ac3cbc..29635b9070fe5 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -279,6 +279,79 @@ jobs: SLACK_TITLE: "Build failure" SLACK_FOOTER: "" + - name: Slack Notification - Success + if: success() && github.ref == 'refs/heads/master' + uses: rtCamp/action-slack-notify@master + env: + SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }} + SLACK_USERNAME: Buildbot + SLACK_TITLE: "Build Success" + SLACK_FOOTER: "" + octavia-cli-build: + # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. + runs-on: ubuntu-latest + name: "Octavia CLI: Build" + timeout-minutes: 90 + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + + # this intentionally does not use restore-keys so we don't mess with gradle caching + - name: Gradle Caching + uses: actions/cache@v2 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + **/.venv + key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }} + + - uses: actions/setup-java@v1 + with: + java-version: "17" + + - uses: actions/setup-node@v1 + with: + node-version: "16.13.0" + + - uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Set up CI Gradle Properties + run: | + mkdir -p ~/.gradle/ + cat > ~/.gradle/gradle.properties < Date: Thu, 30 Dec 2021 11:03:41 +0100 Subject: [PATCH 12/22] try separate worflow for octavia --- .github/workflows/gradle.yml | 4 -- .github/workflows/octavia-cli.yml | 83 +++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/octavia-cli.yml diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 29635b9070fe5..6ba8e80951169 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -216,10 +216,6 @@ jobs: with: node-version: "16.13.0" - - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - name: Set up CI Gradle Properties run: | mkdir -p ~/.gradle/ diff --git a/.github/workflows/octavia-cli.yml b/.github/workflows/octavia-cli.yml new file mode 100644 index 0000000000000..4f9e6805ef110 --- /dev/null +++ b/.github/workflows/octavia-cli.yml @@ -0,0 +1,83 @@ +name: Octavia CLI CI + +on: + push: + branches: + - '*octavia*' + - '!master' +jobs: + ## Gradle Build (Connectors Base) + # In case of self-hosted EC2 errors, remove this block. + octavia-cli-build: + # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. + runs-on: ubuntu-latest + name: "Octavia CLI: Build" + timeout-minutes: 90 + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + + # this intentionally does not use restore-keys so we don't mess with gradle caching + - name: Gradle Caching + uses: actions/cache@v2 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + **/.venv + key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }} + + - uses: actions/setup-java@v1 + with: + java-version: "17" + + - uses: actions/setup-node@v1 + with: + node-version: "16.13.0" + + - uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Set up CI Gradle Properties + run: | + mkdir -p ~/.gradle/ + cat > ~/.gradle/gradle.properties < Date: Thu, 30 Dec 2021 11:13:04 +0100 Subject: [PATCH 13/22] declared shared deps between connector_bases and platform sub builds --- .github/workflows/octavia-cli.yml | 83 ------------------------------- settings.gradle | 45 +++++++++-------- 2 files changed, 24 insertions(+), 104 deletions(-) delete mode 100644 .github/workflows/octavia-cli.yml diff --git a/.github/workflows/octavia-cli.yml b/.github/workflows/octavia-cli.yml deleted file mode 100644 index 4f9e6805ef110..0000000000000 --- a/.github/workflows/octavia-cli.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Octavia CLI CI - -on: - push: - branches: - - '*octavia*' - - '!master' -jobs: - ## Gradle Build (Connectors Base) - # In case of self-hosted EC2 errors, remove this block. - octavia-cli-build: - # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. - runs-on: ubuntu-latest - name: "Octavia CLI: Build" - timeout-minutes: 90 - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - # this intentionally does not use restore-keys so we don't mess with gradle caching - - name: Gradle Caching - uses: actions/cache@v2 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - **/.venv - key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }} - - - uses: actions/setup-java@v1 - with: - java-version: "17" - - - uses: actions/setup-node@v1 - with: - node-version: "16.13.0" - - - uses: actions/setup-python@v2 - with: - python-version: "3.8" - - - name: Set up CI Gradle Properties - run: | - mkdir -p ~/.gradle/ - cat > ~/.gradle/gradle.properties < Date: Thu, 30 Dec 2021 11:24:57 +0100 Subject: [PATCH 14/22] better shared deps --- settings.gradle | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index e1ac7cfee2b45..209998b7c8d75 100644 --- a/settings.gradle +++ b/settings.gradle @@ -30,12 +30,14 @@ if (!System.getenv().containsKey("SUB_BUILD")) { } } +// shared +include ':airbyte-commons' + // shared by CONNECTORS_BASE and PLATFORM sub builds if (!System.getenv().containsKey("SUB_BUILD") == "CONNECTORS_BASE" || System.getenv().get("SUB_BUILD") == "PLATFORM") { include ':airbyte-api' include ':airbyte-commons-cli' include ':airbyte-commons-docker' - include ':airbyte-commons' include ':airbyte-config:models' // reused by acceptance tests in connector base. include ':airbyte-db:lib' // reused by acceptance tests in connector base. include ':airbyte-json-validation' From c1272faf827dfa5ae9b3c4c5c5e745abfa9ad2c8 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Thu, 30 Dec 2021 11:39:00 +0100 Subject: [PATCH 15/22] add missing end line --- octavia-cli/.python-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octavia-cli/.python-version b/octavia-cli/.python-version index 55e6aabe60527..89a1ad7ad3cad 100644 --- a/octavia-cli/.python-version +++ b/octavia-cli/.python-version @@ -1 +1 @@ -3.8.12 \ No newline at end of file +3.8.12 From 843ee00955804163e2a5aa0f2fc08b3290c1399d Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Fri, 31 Dec 2021 18:31:25 +0800 Subject: [PATCH 16/22] Does this work? --- settings.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/settings.gradle b/settings.gradle index 209998b7c8d75..902461b97daa5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -34,7 +34,7 @@ if (!System.getenv().containsKey("SUB_BUILD")) { include ':airbyte-commons' // shared by CONNECTORS_BASE and PLATFORM sub builds -if (!System.getenv().containsKey("SUB_BUILD") == "CONNECTORS_BASE" || System.getenv().get("SUB_BUILD") == "PLATFORM") { +//if (!System.getenv().containsKey("SUB_BUILD") == "CONNECTORS_BASE" || System.getenv().get("SUB_BUILD") == "PLATFORM") { include ':airbyte-api' include ':airbyte-commons-cli' include ':airbyte-commons-docker' @@ -55,7 +55,7 @@ if (!System.getenv().containsKey("SUB_BUILD") == "CONNECTORS_BASE" || System.get include ':airbyte-notification' // transitively used by airbyte-workers. include ':airbyte-scheduler:models' // transitively used by airbyte-workers. include ':airbyte-scheduler:persistence' // used by airbyte-workers. -} +//} // platform From 8d1803b7d8afb07ce592fffd856aa01041f7146d Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Fri, 31 Dec 2021 18:42:04 +0800 Subject: [PATCH 17/22] We do need format. --- .github/workflows/gradle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 6ba8e80951169..741c35d9c1330 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -335,8 +335,8 @@ jobs: - name: Build run: | - SUB_BUILD=OCTAVIA_CLI ./gradlew build javadoc --scan - + SUB_BUILD=OCTAVIA_CLI ./gradlew :octavia-cli:build javadoc --scan + - name: Slack Notification - Failure if: failure() && github.ref == 'refs/heads/master' uses: rtCamp/action-slack-notify@master From 381a31e08187d40c326013532ff9c0ee522fa3dc Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Fri, 31 Dec 2021 18:50:00 +0800 Subject: [PATCH 18/22] Does this work without the npm dependencies? --- .github/workflows/gradle.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 741c35d9c1330..316b71515d3d4 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -284,7 +284,6 @@ jobs: SLACK_TITLE: "Build Success" SLACK_FOOTER: "" octavia-cli-build: - # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line. runs-on: ubuntu-latest name: "Octavia CLI: Build" timeout-minutes: 90 @@ -306,10 +305,6 @@ jobs: with: java-version: "17" - - uses: actions/setup-node@v1 - with: - node-version: "16.13.0" - - uses: actions/setup-python@v2 with: python-version: "3.8" From 309b177239df8d9861df9dade5997752648176bf Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Fri, 31 Dec 2021 19:03:17 +0800 Subject: [PATCH 19/22] Clean up build.gradle. Add better warning. --- settings.gradle | 47 ++++++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/settings.gradle b/settings.gradle index 902461b97daa5..428fd49d9e9cd 100644 --- a/settings.gradle +++ b/settings.gradle @@ -19,14 +19,14 @@ sourceControl { rootProject.name = 'airbyte' -// SUB_BUILD is an enum of , PLATFORM, CONNECTORS_BASE. Blank is equivalent to all. +// SUB_BUILD is an enum of , PLATFORM, CONNECTORS_BASE, OCTAVIA_CLI. Blank is equivalent to all. if (!System.getenv().containsKey("SUB_BUILD")) { println("Building all of Airbyte.") } else { def subBuild = System.getenv().get("SUB_BUILD") println("Building Airbyte Sub Build: " + subBuild) if (subBuild != "PLATFORM" && subBuild != "CONNECTORS_BASE" && subBuild != "OCTAVIA_CLI") { - throw new IllegalArgumentException(String.format("%s is invalid. Must be unset or PLATFORM or CONNECTORS_BASE", subBuild)) + throw new IllegalArgumentException(String.format("%s is invalid. Must be unset or PLATFORM or CONNECTORS_BASE or OCTAVIA_CLI", subBuild)) } } @@ -34,29 +34,26 @@ if (!System.getenv().containsKey("SUB_BUILD")) { include ':airbyte-commons' // shared by CONNECTORS_BASE and PLATFORM sub builds -//if (!System.getenv().containsKey("SUB_BUILD") == "CONNECTORS_BASE" || System.getenv().get("SUB_BUILD") == "PLATFORM") { - include ':airbyte-api' - include ':airbyte-commons-cli' - include ':airbyte-commons-docker' - include ':airbyte-config:models' // reused by acceptance tests in connector base. - include ':airbyte-db:lib' // reused by acceptance tests in connector base. - include ':airbyte-json-validation' - include ':airbyte-metrics' - include ':airbyte-oauth' - include ':airbyte-protocol:models' - include ':airbyte-queue' - include ':airbyte-test-utils' - - // airbyte-workers has a lot of dependencies. - include ':airbyte-workers' // reused by acceptance tests in connector base. - include ':airbyte-analytics' // transitively used by airbyte-workers. - include ':airbyte-config:persistence' // transitively used by airbyte-workers. - include ':airbyte-db:jooq' // transitively used by airbyte-workers. - include ':airbyte-notification' // transitively used by airbyte-workers. - include ':airbyte-scheduler:models' // transitively used by airbyte-workers. - include ':airbyte-scheduler:persistence' // used by airbyte-workers. -//} - +include ':airbyte-api' +include ':airbyte-commons-cli' +include ':airbyte-commons-docker' +include ':airbyte-config:models' // reused by acceptance tests in connector base. +include ':airbyte-db:lib' // reused by acceptance tests in connector base. +include ':airbyte-json-validation' +include ':airbyte-metrics' +include ':airbyte-oauth' +include ':airbyte-protocol:models' +include ':airbyte-queue' +include ':airbyte-test-utils' + +// airbyte-workers has a lot of dependencies. +include ':airbyte-workers' // reused by acceptance tests in connector base. +include ':airbyte-analytics' // transitively used by airbyte-workers. +include ':airbyte-config:persistence' // transitively used by airbyte-workers. +include ':airbyte-db:jooq' // transitively used by airbyte-workers. +include ':airbyte-notification' // transitively used by airbyte-workers. +include ':airbyte-scheduler:models' // transitively used by airbyte-workers. +include ':airbyte-scheduler:persistence' // used by airbyte-workers. // platform if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "PLATFORM") { From fbdaab25b5fd3cadd8e7bb3a3c368dbeb4982ba3 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 3 Jan 2022 09:49:31 +0100 Subject: [PATCH 20/22] comment about python3.8 in gradle.yml --- .github/workflows/gradle.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 316b71515d3d4..adc1ad47b14d9 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -305,6 +305,8 @@ jobs: with: java-version: "17" + # octavia-cli install and testing requires Python. + # We use 3.8 in this project because 3.7 is not supported on Apple M1. - uses: actions/setup-python@v2 with: python-version: "3.8" From 47ebfd61f747c424a7c50a35d737451164b4236d Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 3 Jan 2022 09:50:32 +0100 Subject: [PATCH 21/22] default to Elastic License 2.0 --- octavia-cli/LICENSE | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 octavia-cli/LICENSE diff --git a/octavia-cli/LICENSE b/octavia-cli/LICENSE deleted file mode 100644 index 0e208b4b49e34..0000000000000 --- a/octavia-cli/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2021 Airbyte, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. From 61cf0ae9fc274ea0ba6c6ec09180674757bb1d77 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 4 Jan 2022 10:19:50 +0100 Subject: [PATCH 22/22] re-add MIT --- octavia-cli/LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 octavia-cli/LICENSE diff --git a/octavia-cli/LICENSE b/octavia-cli/LICENSE new file mode 100644 index 0000000000000..ec45d182fcb90 --- /dev/null +++ b/octavia-cli/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Airbyte, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.