🐙 Bootstrap octavia-cli project

.github/workflows/gradle.yml

@@ -275,6 +275,74 @@ jobs:
           SLACK_TITLE: "Build failure"
           SLACK_FOOTER: ""
 
+      - name: Slack Notification - Success
+        if: success() && github.ref == 'refs/heads/master'
+        uses: rtCamp/action-slack-notify@master
+        env:
+          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
+          SLACK_USERNAME: Buildbot
+          SLACK_TITLE: "Build Success"
+          SLACK_FOOTER: ""
+  octavia-cli-build:
+    runs-on: ubuntu-latest
+    name: "Octavia CLI: Build"
+    timeout-minutes: 90
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+
+      # this intentionally does not use restore-keys so we don't mess with gradle caching
+      - name: Gradle Caching
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+            **/.venv
+          key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}
+
+      - uses: actions/setup-java@v1
+        with:
+          java-version: "17"
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+
+      - name: Set up CI Gradle Properties
+        run: |
+          mkdir -p ~/.gradle/
+          cat > ~/.gradle/gradle.properties <<EOF
+          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+          org.gradle.workers.max=8
+          org.gradle.vfs.watch=false
+          EOF
+
+      - name: Format
+        run: SUB_BUILD=OCTAVIA_CLI ./gradlew format --scan --info --stacktrace
+
+      - name: Ensure no file change
+        run: git --no-pager diff && test -z "$(git --no-pager diff)"
+
+      - name: Build
+        run: |
+          SUB_BUILD=OCTAVIA_CLI ./gradlew :octavia-cli:build javadoc --scan
+
+      - name: Slack Notification - Failure
+        if: failure() && github.ref == 'refs/heads/master'
+        uses: rtCamp/action-slack-notify@master
+        env:
+          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
+          SLACK_USERNAME: Buildozer
+          SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
+          SLACK_COLOR: DC143C
+          SLACK_TITLE: "Build failure"
+          SLACK_FOOTER: ""
+
       - name: Slack Notification - Success
         if: success() && github.ref == 'refs/heads/master'
         uses: rtCamp/action-slack-notify@master

.gitignore

@@ -1,5 +1,6 @@
 .gradle
 .idea
+.vscode
 *.iml
 *.swp
 build

octavia-cli/.coveragerc

@@ -0,0 +1,3 @@
+[report]
+# show lines missing coverage
+show_missing = true

octavia-cli/.dockerignore

@@ -0,0 +1,3 @@
+build
+.venv
+octavia_cli.egg-info

octavia-cli/.gitignore

@@ -0,0 +1,2 @@
+.coverage
+.venv

octavia-cli/.python-version

@@ -0,0 +1 @@
+3.8.12

octavia-cli/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.8.12-slim as base
+
+RUN apt-get upgrade \
+    && pip install --upgrade pip
+
+WORKDIR /home/octavia-cli
+COPY . ./
+
+RUN pip install --no-cache-dir .
+
+RUN useradd --create-home --shell /bin/bash octavia-cli
+USER octavia-cli
+
+ENTRYPOINT ["octavia"]
+
+LABEL io.airbyte.version=dev
+LABEL io.airbyte.name=airbyte/octavia-cli

octavia-cli/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Airbyte, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

octavia-cli/README.md

@@ -0,0 +1,55 @@
+# 🐙 Octavia CLI
+
+Octavia CLI is a tool to manage Airbyte configuration in YAML.
+It has the following features:
+* Scaffolding of a readable directory architecture that will host the YAML configs.
+* Auto-generation of YAML config file that matches the resources' schemas.
+* Manage Airbyte resources with YAML config files.
+* Safe resources update through diff display and validation.
+* Simple secret management to avoid versioning credentials.
+
+The project is under development: readers can refer to our [tech spec deck](https://docs.google.com/presentation/d/10RjkCzBiVhCivnjSh63icYI7wG6S0N0ZIErEIsmXTqM/edit?usp=sharing) for an introduction to the tool.
+
+# Usage
+We encourage users to use the CLI with docker to avoid the hassle of setting up a Python installation. 
+The project is under development: we have not yet published any docker image to our Docker registry.
+
+1. Build the image locally:
+```bash
+docker build -t octavia-cli:dev --rm .
+```
+2. Run the CLI from docker:
+```bash
+docker run octavia-cli:dev 
+````
+3. Create an `octavia` alias in your `.bashrc` or `.zshrc`: 
+````bash
+echo 'alias octavia="docker run octavia-cli:dev"'  >> ~/.zshrc
+source ~/.zshrc
+octavia
+````
+
+# Current development status
+Octavia is currently under development. 
+You can find a detailed and updated execution plan [here](https://docs.google.com/spreadsheets/d/1weB9nf0Zx3IR_QvpkxtjBAzyfGb7B0PWpsVt6iMB5Us/edit#gid=0).
+We welcome community contributions!
+
+Summary of achievements:
+
+| Date       | Milestone                           |
+|------------|-------------------------------------|
+| 2021-12-22 | Bootstrapping the project's code base |
+
+# Developing locally
+1. Install Python 3.10.0. We suggest doing it through `pyenv`
+2. Create a virtualenv: `python -m venv .venv`
+3. Activate the virtualenv: `source .venv/bin/activate`
+4. Install dev dependencies: `pip install -e .\[dev\]`
+5. Install `pre-commit` hooks: `pre-commit install`
+6. Run the test suite: `pytest --cov=octavia_cli unit_tests`
+7. Iterate; please check the [Contributing](#contributing) for instructions on contributing.
+
+# Contributing
+1. Please sign up to [Airbyte's Slack workspace](https://slack.airbyte.io/) and join the `#octavia-cli`. We'll sync up community efforts in this channel.
+2. Read the [execution plan](https://docs.google.com/spreadsheets/d/1weB9nf0Zx3IR_QvpkxtjBAzyfGb7B0PWpsVt6iMB5Us/edit#gid=0) and find a task you'd like to work on.
+3. Open a PR, make sure to test your code thoroughly. 

octavia-cli/build.gradle

@@ -0,0 +1,9 @@
+plugins {
+    id 'airbyte-python'
+    id 'airbyte-docker'
+}
+
+airbytePython {
+    moduleDirectory 'octavia_cli'
+}
+

octavia-cli/octavia_cli/entrypoint.py

@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+import click
+
+
+@click.group()
+@click.option("--airbyte-url", envvar="AIRBYTE_URL", default="http://localhost:8000", help="The URL of your Airbyte instance.")
+def octavia(airbyte_url):
+    # TODO: check if the airbyte_url is reachable
+    click.secho(f"🐙 - Octavia is targetting your Airbyte instance running at {airbyte_url}")
+
+
+@octavia.command(help="Scaffolds a local project directories.")
+def init():
+    raise click.ClickException("The init command is not yet implemented.")
+
+
+@octavia.command(name="list", help="List existing resources on the Airbyte instance.")
+def _list():
+    raise click.ClickException("The init command is not yet implemented.")
+
+
+@octavia.command(name="import", help="Import an existing resources from the Airbyte instance.")
+def _import():
+    raise click.ClickException("The init command is not yet implemented.")
+
+
+@octavia.command(help="Generate a YAML configuration file to manage a resource.")
+def create():
+    raise click.ClickException("The init command is not yet implemented.")
+
+
+@octavia.command(help="Create or update resources according to YAML configurations.")
+def apply():
+    raise click.ClickException("The init command is not yet implemented.")
+
+
+@octavia.command(help="Delete resources")
+def delete():
+    raise click.ClickException("The init command is not yet implemented.")

octavia-cli/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+log_cli = 1
+log_cli_level = INFO
+log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)
+log_cli_date_format=%Y-%m-%d %H:%M:%S

octavia-cli/setup.py

@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+import pathlib
+
+from setuptools import find_packages, setup
+
+# The directory containing this file
+HERE = pathlib.Path(__file__).parent
+
+# The text of the README file
+README = (HERE / "README.md").read_text()
+
+setup(
+    name="octavia-cli",
+    version="0.1.0",
+    description="A command line interface to manage Airbyte configurations",
+    long_description=README,
+    author="Airbyte",
+    author_email="contact@airbyte.io",
+    license="MIT",
+    url="https://github.com/airbytehq/airbyte",
+    classifiers=[
+        # This information is used when browsing on PyPi.
+        # Dev Status
+        "Development Status :: 3 - Alpha",
+        # Project Audience
+        "Intended Audience :: Developers",
+        "Topic :: Scientific/Engineering",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        "License :: OSI Approved :: MIT License",
+        # Python Version Support
+        "Programming Language :: Python :: 3.10",
+    ],
+    keywords="airbyte cli command-line-interface configuration",
+    project_urls={
+        "Documentation": "https://docs.airbyte.io/",
+        "Source": "https://github.com/airbytehq/airbyte",
+        "Tracker": "https://github.com/airbytehq/airbyte/issues",
+    },
+    packages=find_packages(exclude=("tests", "docs")),
+    install_requires=["click~=8.0.3"],
+    python_requires=">=3.8.12",
+    extras_require={
+        "dev": ["MyPy~=0.812", "pytest~=6.2.5", "pytest-cov", "pytest-mock", "requests-mock", "pre-commit"],
+        "sphinx-docs": [
+            "Sphinx~=4.2",
+            "sphinx-rtd-theme~=1.0",
+        ],
+    },
+    entry_points={"console_scripts": ["octavia=octavia_cli.entrypoint:octavia"]},
+)

octavia-cli/unit_tests/test_entrypoint.py

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+import pytest
+from click.testing import CliRunner
+from octavia_cli import entrypoint
+
+
+def test_octavia():
+    runner = CliRunner()
+    result = runner.invoke(entrypoint.octavia)
+    assert result.exit_code == 0
+    assert result.output.startswith("Usage: octavia [OPTIONS] COMMAND [ARGS]...")
+
+
+@pytest.mark.parametrize(
+    "command",
+    [entrypoint.init, entrypoint.apply, entrypoint.create, entrypoint.delete, entrypoint._list, entrypoint._import],
+)
+def test_not_implemented_commands(command):
+    runner = CliRunner()
+    result = runner.invoke(command)
+    assert result.exit_code == 1
+    assert result.output.endswith("not yet implemented.\n")

settings.gradle

@@ -19,27 +19,30 @@ sourceControl {
 
 rootProject.name = 'airbyte'
 
-// SUB_BUILD is an enum of <blank>, PLATFORM, CONNECTORS_BASE. Blank is equivalent to all.
+// SUB_BUILD is an enum of <blank>, PLATFORM, CONNECTORS_BASE, OCTAVIA_CLI. Blank is equivalent to all.
 if (!System.getenv().containsKey("SUB_BUILD")) {
     println("Building all of Airbyte.")
 } else {
     def subBuild = System.getenv().get("SUB_BUILD")
     println("Building Airbyte Sub Build: " + subBuild)
-    if (subBuild != "PLATFORM" && subBuild != "CONNECTORS_BASE") {
-        throw new IllegalArgumentException(String.format("%s is invalid. Must be unset or PLATFORM or CONNECTORS_BASE", subBuild))
+    if (subBuild != "PLATFORM" && subBuild != "CONNECTORS_BASE" && subBuild != "OCTAVIA_CLI") {
+        throw new IllegalArgumentException(String.format("%s is invalid. Must be unset or PLATFORM or CONNECTORS_BASE or OCTAVIA_CLI", subBuild))
     }
 }
 
 // shared
-include ':airbyte-api'
 include ':airbyte-commons'
-include ':airbyte-commons-docker'
+
+// shared by CONNECTORS_BASE and PLATFORM sub builds
+include ':airbyte-api'
 include ':airbyte-commons-cli'
+include ':airbyte-commons-docker'
 include ':airbyte-config:models' // reused by acceptance tests in connector base.
 include ':airbyte-db:lib' // reused by acceptance tests in connector base.
 include ':airbyte-json-validation'
-include ':airbyte-protocol:models'
 include ':airbyte-metrics'
+include ':airbyte-oauth'
+include ':airbyte-protocol:models'
 include ':airbyte-queue'
 include ':airbyte-test-utils'
 
@@ -60,7 +63,6 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD"
     include ':airbyte-config:specs'
     include ':airbyte-container-orchestrator'
     include ':airbyte-webapp-e2e-tests'
-    include ':airbyte-oauth'
     include ':airbyte-scheduler:app'
     include ':airbyte-scheduler:client'
     include ':airbyte-server'
@@ -86,8 +88,6 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD"
     include ':airbyte-integrations:connector-templates:generator'
     include ':airbyte-integrations:bases:debezium'
 
-    include ':airbyte-oauth'
-
     // Needed by normalization integration tests
     include ':airbyte-integrations:connectors:destination-bigquery'
     include ':airbyte-integrations:connectors:destination-jdbc'
@@ -105,6 +105,10 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD"
     include ':tools:code-generator'
 }
 
+if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "OCTAVIA_CLI") {
+    include ':octavia-cli'
+}
+
 // connectors
 if (!System.getenv().containsKey("SUB_BUILD")) {
     // include all connector projects