🐙 octavia-cli: generate open api client

.pre-commit-config.yaml

@@ -1,25 +1,22 @@
 default_language_version:
-  python: python3.7
+  python: python3
 
 repos:
   - repo: https://github.com/johann-petrak/licenseheaders.git
     rev: v0.8.8
     hooks:
       - id: licenseheaders
         args: ["--tmpl=LICENSE_SHORT", "--ext=py", "-f"]
-
   - repo: https://github.com/ambv/black
     rev: 21.11b1
     hooks:
       - id: black
-
   - repo: https://github.com/timothycrosley/isort
     rev: 5.10.1
     hooks:
       - id: isort
         args: ["--dont-follow-links", "--jobs=-1"]
         additional_dependencies: ["colorama"]
-
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v2.5.0
     hooks:
@@ -29,7 +26,8 @@ repos:
           (?x)^.*(
               .github/|
               source_specs.yaml|
-              destination_specs.yaml
+              destination_specs.yaml|
+              .gitlab-ci.yml
           ).?$
 
   - repo: https://github.com/csachs/pyproject-flake8
@@ -38,12 +36,14 @@ repos:
       - id: pyproject-flake8
         additional_dependencies: ["mccabe"]
         alias: flake8
-
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.910-1
     hooks:
       - id: mypy
-
+        exclude: |
+          (?x)^.*(
+              octavia-cli/unit_tests/|
+          ).?$
   - repo: local
     hooks:
       - id: spec-linter

airbyte-api/src/main/openapi/config.yaml

@@ -1944,8 +1944,10 @@ components:
             $ref: "#/components/schemas/Notification"
         firstCompletedSync:
           type: boolean
+          nullable: true
         feedbackDone:
           type: boolean
+          nullable: true
     WorkspaceUpdate:
       type: object
       required:

octavia-cli/.gitignore

@@ -1,2 +1,3 @@
 .coverage
 .venv
+airbyte_api_client/

octavia-cli/README.md

@@ -14,9 +14,9 @@ The project is under development: readers can refer to our [tech spec deck](http
 We encourage users to use the CLI with docker to avoid the hassle of setting up a Python installation. 
 The project is under development: we have not yet published any docker image to our Docker registry.
 
-1. Build the image locally:
+1. Build the image locally (from the root of the repo):
 ```bash
-docker build -t octavia-cli:dev --rm .
+SUB_BUILD=OCTAVIA_CLI ./gradlew build #from the root of the repo
 ```
 2. Run the CLI from docker:
 ```bash
@@ -38,10 +38,11 @@ Summary of achievements:
 
 | Date       | Milestone                           |
 |------------|-------------------------------------|
+| 2022-01-06 | Generate an API Python client from our Open API spec |
 | 2021-12-22 | Bootstrapping the project's code base |
 
 # Developing locally
-1. Install Python 3.10.0. We suggest doing it through `pyenv`
+1. Install Python 3.8.12. We suggest doing it through `pyenv`
 2. Create a virtualenv: `python -m venv .venv`
 3. Activate the virtualenv: `source .venv/bin/activate`
 4. Install dev dependencies: `pip install -e .\[dev\]`

octavia-cli/bin/generate-api-client.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -e
+
+[ -z "$ROOT_DIR" ] && exit 1
+
+OPENAPI_CONFIG_PATH=airbyte-api/src/main/openapi/config.yaml
+GENERATED_CLIENT_PATH=octavia-cli/airbyte_api_client
+
+function main() {
+  rm -rf "$ROOT_DIR/$GENERATED_CLIENT_PATH"/*.py
+
+  docker run --user "$(id -u):$(id -g)" -v "$ROOT_DIR":/airbyte openapitools/openapi-generator-cli generate \
+    -i "/airbyte/$OPENAPI_CONFIG_PATH" \
+    -o "/airbyte/$GENERATED_CLIENT_PATH" \
+    -g python \
+    --additional-properties=packageName=airbyte_api_client
+}
+
+main "$@"

octavia-cli/build.gradle

@@ -7,3 +7,12 @@ airbytePython {
     moduleDirectory 'octavia_cli'
 }
 
+task generateApiClient(type: Exec) {
+    environment 'ROOT_DIR', rootDir.absolutePath
+    commandLine 'bin/generate-api-client.sh'
+}
+
+blackFormat.dependsOn generateApiClient
+isortFormat.dependsOn generateApiClient
+flakeCheck.dependsOn generateApiClient
+installReqs.dependsOn generateApiClient

octavia-cli/octavia_cli/entrypoint.py

@@ -2,41 +2,53 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+import airbyte_api_client
 import click
+from airbyte_api_client.api import workspace_api
 
 
 @click.group()
 @click.option("--airbyte-url", envvar="AIRBYTE_URL", default="http://localhost:8000", help="The URL of your Airbyte instance.")
-def octavia(airbyte_url):
-    # TODO: check if the airbyte_url is reachable
-    click.secho(f"🐙 - Octavia is targetting your Airbyte instance running at {airbyte_url}")
+@click.pass_context
+def octavia(ctx: click.Context, airbyte_url: str) -> None:
+    ctx.ensure_object(dict)
+    client_configuration = airbyte_api_client.Configuration(host=f"{airbyte_url}/api")
+    api_client = airbyte_api_client.ApiClient(client_configuration)
+    # TODO alafanechere workspace check might deserve its own function
+    api_instance = workspace_api.WorkspaceApi(api_client)
+    api_response = api_instance.list_workspaces()
+    # TODO alafanechere prompt user to chose a workspace if multiple workspaces exist
+    workspace_id = api_response.workspaces[0].workspace_id
+    click.echo(f"🐙 - Octavia is targetting your Airbyte instance running at {airbyte_url} on workspace {workspace_id}")
+    ctx.obj["API_CLIENT"] = api_client
+    ctx.obj["WORKSPACE_ID"] = workspace_id
 
 
 @octavia.command(help="Scaffolds a local project directories.")
-def init():
+def init() -> None:
     raise click.ClickException("The init command is not yet implemented.")
 
 
 @octavia.command(name="list", help="List existing resources on the Airbyte instance.")
-def _list():
-    raise click.ClickException("The init command is not yet implemented.")
+def _list() -> None:
+    raise click.ClickException("The list command is not yet implemented.")
 
 
 @octavia.command(name="import", help="Import an existing resources from the Airbyte instance.")
-def _import():
-    raise click.ClickException("The init command is not yet implemented.")
+def _import() -> None:
+    raise click.ClickException("The import command is not yet implemented.")
 
 
 @octavia.command(help="Generate a YAML configuration file to manage a resource.")
-def create():
-    raise click.ClickException("The init command is not yet implemented.")
+def create() -> None:
+    raise click.ClickException("The create command is not yet implemented.")
 
 
 @octavia.command(help="Create or update resources according to YAML configurations.")
-def apply():
-    raise click.ClickException("The init command is not yet implemented.")
+def apply() -> None:
+    raise click.ClickException("The apply command is not yet implemented.")
 
 
 @octavia.command(help="Delete resources")
-def delete():
-    raise click.ClickException("The init command is not yet implemented.")
+def delete() -> None:
+    raise click.ClickException("The delete command is not yet implemented.")

octavia-cli/setup.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+import os
 import pathlib
 
 from setuptools import find_packages, setup
@@ -39,8 +40,8 @@
         "Source": "https://github.com/airbytehq/airbyte",
         "Tracker": "https://github.com/airbytehq/airbyte/issues",
     },
-    packages=find_packages(exclude=("tests", "docs")),
-    install_requires=["click~=8.0.3"],
+    packages=find_packages(exclude=("unit_tests", "docs")),
+    install_requires=["click~=8.0.3", f"airbyte_api_client @ file://{os.getcwd()}/airbyte_api_client"],
     python_requires=">=3.8.12",
     extras_require={
         "dev": ["MyPy~=0.812", "pytest~=6.2.5", "pytest-cov", "pytest-mock", "requests-mock", "pre-commit"],

octavia-cli/unit_tests/test_entrypoint.py

@@ -2,16 +2,37 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+from unittest import mock
+
+import click
 import pytest
 from click.testing import CliRunner
 from octavia_cli import entrypoint
 
 
-def test_octavia():
+@click.command()
+@click.pass_context
+def dumb(ctx):
+    pass
+
+
+@mock.patch("octavia_cli.entrypoint.workspace_api")
+@mock.patch("octavia_cli.entrypoint.airbyte_api_client")
+def test_octavia(mock_airbyte_api_client: mock.Mock, mock_workspace_api: mock.Mock):
+    context_object = {}
+    mock_api_instance = mock_workspace_api.WorkspaceApi.return_value
+    mock_api_instance.list_workspaces.return_value = mock.MagicMock(workspaces=[mock.MagicMock(workspace_id="expected_workspace_id")])
+
+    entrypoint.octavia.add_command(dumb)
     runner = CliRunner()
-    result = runner.invoke(entrypoint.octavia)
+    result = runner.invoke(entrypoint.octavia, ["--airbyte-url", "test-airbyte-url", "dumb"], obj=context_object)
+    mock_airbyte_api_client.Configuration.assert_called_with(host="test-airbyte-url/api")
+    mock_airbyte_api_client.ApiClient.assert_called_with(mock_airbyte_api_client.Configuration.return_value)
+    mock_workspace_api.WorkspaceApi.assert_called_with(mock_airbyte_api_client.ApiClient.return_value)
+    mock_api_instance.list_workspaces.assert_called_once()
+    assert context_object["API_CLIENT"] == mock_airbyte_api_client.ApiClient.return_value
+    assert context_object["WORKSPACE_ID"] == "expected_workspace_id"
     assert result.exit_code == 0
-    assert result.output.startswith("Usage: octavia [OPTIONS] COMMAND [ARGS]...")
 
 
 @pytest.mark.parametrize(

tools/python/.flake8

@@ -5,6 +5,7 @@ exclude =
     .eggs # python libraries"
     .tox
     build
+    airbyte_api_client # generated api client in octavia-cli
 extend-ignore =
   E203,  # whitespace before ':' (conflicts with Black)
   E231,  # Bad trailing comma (conflicts with Black)