Generate spec-first FastAPI Server (#18886)

airbyte-connector-builder-server/src/main/openapi/openapi.yaml

@@ -60,7 +60,7 @@ paths:
   /v1/manifest_template:
     get:
       summary: Return a connector manifest template to use as the default value for the yaml editor
-      operationId: template
+      operationId: getManifestTemplate
       responses:
         "200":
           description: Successful operation

airbyte-connector-builder/.coveragerc

@@ -0,0 +1,3 @@
+[report]
+# show lines missing coverage
+show_missing = true

airbyte-connector-builder/.dockerignore

@@ -0,0 +1,4 @@
+build
+!build/airbyte_api_client
+.venv
+connector_builder.egg-info

airbyte-connector-builder/.gitignore

@@ -0,0 +1,3 @@
+.coverage
+.venv
+state_*.yaml

airbyte-connector-builder/.python-version

@@ -0,0 +1 @@
+3.9.11

airbyte-connector-builder/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.9-slim as base
+
+RUN apt-get upgrade \
+    && pip install --upgrade pip
+
+WORKDIR /home/connector-builder
+COPY . ./
+
+RUN pip install --no-cache-dir .
+
+ENTRYPOINT ["uvicorn", "connector_builder.entrypoint:app", "--host", "0.0.0.0", "--port", "80"]
+
+LABEL io.airbyte.version=0.40.15
+LABEL io.airbyte.name=airbyte/connector-builder
+

airbyte-connector-builder/README.md

@@ -0,0 +1,29 @@
+# Connector builder
+
+
+## Getting started 
+
+Set up the virtual environment and install dependencies
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install .
+```
+
+Then run the server
+```bash
+uvicorn connector_builder.entrypoint:app --host 0.0.0.0 --port 8080
+```
+
+The server is now reachable on localhost:8080
+
+### OpenAPI generation
+
+```bash
+openapi-generator generate -i ../connector-builder-server/src/main/openapi/openapi.yaml -g python-fastapi -c openapi/generator_config.yaml  -o build/server -t openapi/templates 
+```
+
+Or you can run it via Gradle by running this from the Airbyte project root: 
+```bash
+./gradlew :airbyte-connector-builder:generateOpenApiPythonServer
+```

airbyte-connector-builder/build.gradle

@@ -0,0 +1,45 @@
+import org.openapitools.generator.gradle.plugin.tasks.GenerateTask
+
+plugins {
+    id "org.openapi.generator" version "5.3.1"
+    id 'airbyte-python'
+    id 'airbyte-docker'
+}
+
+airbytePython {
+    moduleDirectory 'connector_builder'
+}
+
+task generateOpenApiPythonServer(type: GenerateTask){
+    outputs.upToDateWhen { false }
+
+    def generatedCodeDir = "$buildDir/server"
+    inputSpec = "$rootDir.absolutePath/airbyte-connector-builder/src/main/openapi/openapi.yaml"
+    outputDir = generatedCodeDir
+
+    generatorName = "python-fastapi"
+    configFile = "$projectDir/openapi/generator_config.yaml"
+    templateDir = "$projectDir/openapi/templates"
+    packageName = "connector_builder.generated"
+
+    // After we generate, we're only interested in the API declaration and the generated pydantic models.
+    // So we copy those from the build/ directory
+    doLast {
+        def sourceDir = "$generatedCodeDir/src/connector_builder/generated/"
+        def targetDir = "$projectDir/connector_builder/generated"
+        mkdir targetDir
+        copy {
+            from "$sourceDir/apis"
+            include "*_interface.py", "__init__.py"
+            into "$targetDir/apis"
+        }
+        copy {
+            from "$sourceDir/models"
+            include "*.py"
+            into "$targetDir/models"
+        }
+    }
+}
+
+project.build.dependsOn(generateOpenApiPythonServer)
+

airbyte-connector-builder/connector_builder/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#

airbyte-connector-builder/connector_builder/entrypoint.py

@@ -0,0 +1,15 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from connector_builder.generated.apis.default_api_interface import initialize_router
+from connector_builder.impl.default_api import DefaultApiImpl
+from fastapi import FastAPI
+
+app = FastAPI(
+    title="Connector Builder Server API",
+    description="Connector Builder Server API ",
+    version="1.0.0",
+)
+
+app.include_router(initialize_router(DefaultApiImpl()))

airbyte-connector-builder/connector_builder/generated/apis/default_api_interface.py

@@ -0,0 +1,145 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+import inspect
+from abc import ABC, abstractmethod
+from typing import Callable, Dict, List  # noqa: F401
+
+from fastapi import (  # noqa: F401
+    APIRouter,
+    Body,
+    Cookie,
+    Depends,
+    Form,
+    Header,
+    Path,
+    Query,
+    Response,
+    Security,
+    status,
+)
+
+from connector_builder.generated.models.extra_models import TokenModel  # noqa: F401
+
+
+from connector_builder.generated.models.invalid_input_exception_info import InvalidInputExceptionInfo
+from connector_builder.generated.models.known_exception_info import KnownExceptionInfo
+from connector_builder.generated.models.stream_read import StreamRead
+from connector_builder.generated.models.stream_read_request_body import StreamReadRequestBody
+from connector_builder.generated.models.streams_list_read import StreamsListRead
+from connector_builder.generated.models.streams_list_request_body import StreamsListRequestBody
+
+
+class DefaultApi(ABC):
+    """
+    NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+    Do not edit the class manually.
+    """
+
+    @abstractmethod
+    async def get_manifest_template(
+        self, 
+    ) -> str:
+        """
+        Return a connector manifest template to use as the default value for the yaml editor
+        """
+
+    @abstractmethod
+    async def list_streams(
+        self, 
+        streams_list_request_body: StreamsListRequestBody = Body(None, description=""),
+    ) -> StreamsListRead:
+        """
+        List all streams present in the connector manifest, along with their specific request URLs
+        """
+
+    @abstractmethod
+    async def read_stream(
+        self, 
+        stream_read_request_body: StreamReadRequestBody = Body(None, description=""),
+    ) -> StreamRead:
+        """
+        Reads a specific stream in the source. TODO in a later phase - only read a single slice of data.
+        """
+
+
+def _assert_signature_is_set(method: Callable) -> None:
+    """
+    APIRouter().add_api_route expects the input method to have a signature. It gets signatures
+    by running inspect.signature(method) under the hood. 
+
+    In the case that an instance method does not declare "self" as an input parameter (due to developer error 
+    for example), then the call to inspect.signature() raises a ValueError and fails.
+
+    Ideally, we'd automatically detect & correct this problem. To do that, we'd need to do
+    setattr(method, "__signature__", <correct_signature>) but that's not possible because instance
+    methods (i.e the input to this function) are object subclasses, and you can't use setattr on objects
+    (https://stackoverflow.com/a/12839070/3237889)
+
+    The workaround this method implements is to raise an exception at runtime if the input method fails
+    when inspect.signature() is called. This is good enough because the error will be detected
+    immediately when the developer tries to run the server, so builds should very quickly fail and this
+    will practically never make it to a production scenario.
+    """
+    try:
+        inspect.signature(method)
+    except ValueError as e:
+        # Based on empirical observation, the call to inspect fails with a ValueError
+        # with exactly one argument: "invalid method signature"
+        if e.args and len(e.args) == 1 and e.args[0] == "invalid method signature":
+            # I couldn't figure out how to setattr on a "method" object to populate the signature. For now just kick
+            # it back to the developer and tell them to set the "self" variable
+            raise Exception(f"Method {method.__name__} in class {type(method.__self__).__name__} must declare the variable 'self'. ")
+        else:
+            raise
+
+
+def initialize_router(api: DefaultApi) -> APIRouter:
+    router = APIRouter()
+
+    _assert_signature_is_set(api.get_manifest_template)
+    router.add_api_route(
+        "/v1/manifest_template",
+        endpoint=api.get_manifest_template,
+        methods=["GET"],
+        responses={
+            200: {"model": str, "description": "Successful operation"},
+        },
+        tags=["default"],
+        summary="Return a connector manifest template to use as the default value for the yaml editor",
+        response_model_by_alias=True,
+    )
+
+    _assert_signature_is_set(api.list_streams)
+    router.add_api_route(
+        "/v1/streams/list",
+        endpoint=api.list_streams,
+        methods=["POST"],
+        responses={
+            200: {"model": StreamsListRead, "description": "Successful operation"},
+            400: {"model": KnownExceptionInfo, "description": "Exception occurred; see message for details."},
+            422: {"model": InvalidInputExceptionInfo, "description": "Input failed validation"},
+        },
+        tags=["default"],
+        summary="List all streams present in the connector manifest, along with their specific request URLs",
+        response_model_by_alias=True,
+    )
+
+    _assert_signature_is_set(api.read_stream)
+    router.add_api_route(
+        "/v1/stream/read",
+        endpoint=api.read_stream,
+        methods=["POST"],
+        responses={
+            200: {"model": StreamRead, "description": "Successful operation"},
+            400: {"model": KnownExceptionInfo, "description": "Exception occurred; see message for details."},
+            422: {"model": InvalidInputExceptionInfo, "description": "Input failed validation"},
+        },
+        tags=["default"],
+        summary="Reads a specific stream in the source. TODO in a later phase - only read a single slice of data.",
+        response_model_by_alias=True,
+    )
+
+
+    return router

airbyte-connector-builder/connector_builder/generated/models/extra_models.py

@@ -0,0 +1,12 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+# coding: utf-8
+
+from pydantic import BaseModel
+
+class TokenModel(BaseModel):
+    """Defines a token model."""
+
+    sub: str

airbyte-connector-builder/connector_builder/generated/models/http_request.py

@@ -0,0 +1,34 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+# coding: utf-8
+
+from __future__ import annotations
+from datetime import date, datetime  # noqa: F401
+
+import re  # noqa: F401
+from typing import Any, Dict, List, Optional  # noqa: F401
+
+from pydantic import AnyUrl, BaseModel, EmailStr, validator  # noqa: F401
+
+
+class HttpRequest(BaseModel):
+    """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+
+    Do not edit the class manually.
+
+    HttpRequest - a model defined in OpenAPI
+
+        url: The url of this HttpRequest.
+        parameters: The parameters of this HttpRequest [Optional].
+        body: The body of this HttpRequest [Optional].
+        headers: The headers of this HttpRequest [Optional].
+    """
+
+    url: str
+    parameters: Optional[Dict[str, Any]] = None
+    body: Optional[Dict[str, Any]] = None
+    headers: Optional[Dict[str, Any]] = None
+
+HttpRequest.update_forward_refs()

airbyte-connector-builder/connector_builder/generated/models/http_response.py

@@ -0,0 +1,32 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+# coding: utf-8
+
+from __future__ import annotations
+from datetime import date, datetime  # noqa: F401
+
+import re  # noqa: F401
+from typing import Any, Dict, List, Optional  # noqa: F401
+
+from pydantic import AnyUrl, BaseModel, EmailStr, validator  # noqa: F401
+
+
+class HttpResponse(BaseModel):
+    """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+
+    Do not edit the class manually.
+
+    HttpResponse - a model defined in OpenAPI
+
+        status: The status of this HttpResponse.
+        body: The body of this HttpResponse [Optional].
+        headers: The headers of this HttpResponse [Optional].
+    """
+
+    status: int
+    body: Optional[Dict[str, Any]] = None
+    headers: Optional[Dict[str, Any]] = None
+
+HttpResponse.update_forward_refs()

airbyte-connector-builder/connector_builder/generated/models/invalid_input_exception_info.py

@@ -0,0 +1,35 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+# coding: utf-8
+
+from __future__ import annotations
+from datetime import date, datetime  # noqa: F401
+
+import re  # noqa: F401
+from typing import Any, Dict, List, Optional  # noqa: F401
+
+from pydantic import AnyUrl, BaseModel, EmailStr, validator  # noqa: F401
+from connector_builder.generated.models.invalid_input_property import InvalidInputProperty
+
+
+class InvalidInputExceptionInfo(BaseModel):
+    """NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+
+    Do not edit the class manually.
+
+    InvalidInputExceptionInfo - a model defined in OpenAPI
+
+        message: The message of this InvalidInputExceptionInfo.
+        exception_class_name: The exception_class_name of this InvalidInputExceptionInfo [Optional].
+        exception_stack: The exception_stack of this InvalidInputExceptionInfo [Optional].
+        validation_errors: The validation_errors of this InvalidInputExceptionInfo.
+    """
+
+    message: str
+    exception_class_name: Optional[str] = None
+    exception_stack: Optional[List[str]] = None
+    validation_errors: List[InvalidInputProperty]
+
+InvalidInputExceptionInfo.update_forward_refs()