From e6de1a57538a4b169cf38b84a2d1be84fcd5a8cc Mon Sep 17 00:00:00 2001 From: akash5100 Date: Wed, 13 Sep 2023 17:58:13 +0530 Subject: [PATCH 1/4] init appNexus the API Layer --- .../tests/appNexus/__init__.py | 0 .../tests/appNexus/test_base.py | 44 +++++ .../utils/appNexus/APILayer.md | 161 ++++++++++++++++++ .../utils/appNexus/__init__.py | 2 + .../utils/appNexus/adapters.py | 1 + .../utils/appNexus/backends.py | 1 + .../contentcuration/utils/appNexus/base.py | 81 +++++++++ 7 files changed, 290 insertions(+) create mode 100644 contentcuration/contentcuration/tests/appNexus/__init__.py create mode 100644 contentcuration/contentcuration/tests/appNexus/test_base.py create mode 100644 contentcuration/contentcuration/utils/appNexus/APILayer.md create mode 100644 contentcuration/contentcuration/utils/appNexus/__init__.py create mode 100644 contentcuration/contentcuration/utils/appNexus/adapters.py create mode 100644 contentcuration/contentcuration/utils/appNexus/backends.py create mode 100644 contentcuration/contentcuration/utils/appNexus/base.py diff --git a/contentcuration/contentcuration/tests/appNexus/__init__.py b/contentcuration/contentcuration/tests/appNexus/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/contentcuration/contentcuration/tests/appNexus/test_base.py b/contentcuration/contentcuration/tests/appNexus/test_base.py new file mode 100644 index 0000000000..0ae614dea4 --- /dev/null +++ b/contentcuration/contentcuration/tests/appNexus/test_base.py @@ -0,0 +1,44 @@ +from contentcuration.utils.appNexus.base import Backend, Adapter + +class MockBackend(Backend): + _instance = None + + def connect(self) -> None: + return super().connect() + + def make_request(self, url: str, params=None): + return super().make_request(url, params) + + def request(self) -> None: + return super().request() + + def response(self) -> None: + return super().response() + + @classmethod + def _create_instance(cls) -> 'MockBackend': + return cls() + + +class MockAdapter(Adapter): + def mockoperation(self): + pass + + +def test_backend_singleton(): + b1, b2 = MockBackend.get_instance(), MockBackend.get_instance() + assert id(b1) == id(b2) + +def test_adapter_creation(): + a = MockAdapter() + assert isinstance(a, Adapter) + +def test_adapter_backend_default(): + b = MockBackend() + adapter = Adapter(backend=b) + assert isinstance(adapter.backend, Backend) + +def test_adapter_backend_custom(): + b = MockBackend() + a = Adapter(backend=b) + assert a.backend is b diff --git a/contentcuration/contentcuration/utils/appNexus/APILayer.md b/contentcuration/contentcuration/utils/appNexus/APILayer.md new file mode 100644 index 0000000000..d5e8e3fa45 --- /dev/null +++ b/contentcuration/contentcuration/utils/appNexus/APILayer.md @@ -0,0 +1,161 @@ +## API Layer Documentation + +### Overview + +Within the `contentcuration` app in Studio, we want to build an API layer that acts as a communication bridge with different backends like Docker Images, Google Cloud Platform's Vertex AI, and VM instances, cloud storage services, etc. The goal is to make sure this API layer can work with these backends, regardless of where or how they do the job. As long as the input and output formats stay the same, this setup provides flexibility in choosing and using backend resources. + +### Description and outcomes + +The stand-alone deployed backend service(s) will not have direct access to `contentcuration` models or the database for that matter, so this API layer facilitates access to these resources by receiving and returning a standardized requests and responses, irrespective of the backend interacted with. + +#### The Architecture + +Screenshot 2023-09-11 at 14 50 06 + +The key components of this architecture are as follows: + +#### 1. Creating the Backend Interface + +The Backend class serves as an abstract interface that outlines the operations all backends must support. It implements the Singleton pattern to ensure that only one instance of each backend type can exist. The methods defined by the Backend class are: + +```python +ABSTRACT CLASS Backend: + _instance = None # Private variable to hold the instance + + ABSTRACT METHOD connect() + # Provides blue print to connect + pass + + ABSTRACT METHOD make_request(params) + # provide blue print to make request + pass + + ABSTRACT METHOD request(params) + # provide blue print for the request object + pass + + ABSTRACT METHOD response(params) + # provides blue print for the response object + pass + + CLASS METHOD get_instance(cls) + IF cls._instance is None: + cls._instance = cls._create_instance() + return cls._instance + + CLASS METHOD _create_instance(cls) + raise NotImplementedError # concrete class must implement +``` + +Different backends can now be created by implementing the base `Backend` class: + +```python +# Implement CONCRETE CLASS using ABSTRACT Backend class +CLASS GCS IMPLEMENTS Backend: + METHOD make_request(params): + # make request to Google Cloud Storage services + + METHOD connect(params): + # Implement the connect method for GCS + + CLASS METHOD _create_instance(cls) + # initialize a GCS Backend instance + +CLASS ML IMPLEMENTS Backend: + METHOD make_request(params): + # make request to DeepLearning models hosted as service + + METHOD connect(params): + # Implement the connect method for hosted ML service + + CLASS METHOD _create_instance(cls) + # initialize a ML Backend instance + +CLASS OtherBackend IMPLEMENTS Backend: + ... + [you get the idea] +``` + +To create an instance of a backend, using the `ML` class as an example, use the `get_instance()` method: + +```python +>>> backend = ML.get_instance() +``` + +To centralize the creation of `Backend` instances based on specific Django settings(e.g. dev vs. production environments), create `BackendFactory` class. This should follow the Factory Design Pattern. + +```python +# Factory to instantiate the Backend based on Django Settings +CLASS BackendFactory: + METHOD create_backend(self, backend=None) -> Backend + IF backend: + return backend + ELSE: + # Create an Adapter instance based on Django settings + IF DjangoSettings is 'SomeSetting': + backend = GCS.get_instance() # Use of Singleton pattern + ELSE IF DjangoSettings is 'AnotherSetting': + backend = ML.get_instance() + ELSE + RAISE ValueError + # Return the created Backend instance + RETURN backend +``` +The `BackendFactory`'s `create_backend` method optionally allows a `Backend` instance to be injected into the factory instead of relying solely on Django settings. This is particularly useful if we want to explicitly specify the backend to use. + +### Creating Adapter that accepts any Backend + +The **`Adapter`** class can be initialized with a `Backend` instance(optional) which provides a `make_request` method that forwards requests to the chosen `Backend`, while adhering to its specific `request` and `response` formats. + +```python +CLASS Adapter: + + METHOD __init__(self, backend(Optional) defaults None) + # Initialize the Backend with BackendFactory + backend_factory = BackendFactory() + SET backend = backend_factory.create_backend(backend) + + METHOD request(self): + # something + return self.backend.request() + + METHOD response(self): + # something + return self.backend.response() +``` + +With this `Adapter` class in place, we can create Adapter that are able interact with any backend we need. + +```python +CLASS Recommendation INHERITS ADAPTER: + METHOD generateEmbeddings(self, params) -> Boolean + # [ Implementation ] + + METHOD getRecommendation(self, params) -> Array + # [ Implementation ] + +CLASS Transcription INHERITS ADAPTER: + METHOD generateCaption(self, params) -> Array + # [ Implementation ] + +CLASS OtherAdapter INHERITS ADAPTER: + METHOD someOperation(self, params) -> Any + # Operation that any backend wants +``` + +Below is a sample use case, using the `ML` backend as an example: + +```python +>>> backend = ML.get_instance() +>>> adapter = Transcription(backend) +``` + +To access specific methods within the adapter: + +```python +>>> adapter.generateCaption(...) +``` + +### Resources + +[OOP Design patterns](https://refactoring.guru/design-patterns/catalog) diff --git a/contentcuration/contentcuration/utils/appNexus/__init__.py b/contentcuration/contentcuration/utils/appNexus/__init__.py new file mode 100644 index 0000000000..09cf2db5d6 --- /dev/null +++ b/contentcuration/contentcuration/utils/appNexus/__init__.py @@ -0,0 +1,2 @@ +from .adapters import * +from .backends import * \ No newline at end of file diff --git a/contentcuration/contentcuration/utils/appNexus/adapters.py b/contentcuration/contentcuration/utils/appNexus/adapters.py new file mode 100644 index 0000000000..c94b712ddf --- /dev/null +++ b/contentcuration/contentcuration/utils/appNexus/adapters.py @@ -0,0 +1 @@ +# A file to implement adapters like Recommendation, Transcription etc diff --git a/contentcuration/contentcuration/utils/appNexus/backends.py b/contentcuration/contentcuration/utils/appNexus/backends.py new file mode 100644 index 0000000000..fc9d771c82 --- /dev/null +++ b/contentcuration/contentcuration/utils/appNexus/backends.py @@ -0,0 +1 @@ +# Implementation of ML, GCS Backend etc. \ No newline at end of file diff --git a/contentcuration/contentcuration/utils/appNexus/base.py b/contentcuration/contentcuration/utils/appNexus/base.py new file mode 100644 index 0000000000..f42a13aa77 --- /dev/null +++ b/contentcuration/contentcuration/utils/appNexus/base.py @@ -0,0 +1,81 @@ +from abc import ABC, abstractmethod +from builtins import NotImplementedError +from typing import Optional, Union, Dict + + +class Backend(ABC): + """ An abstract base class for backend interfaces that also implements the singleton pattern """ + _instance = None + + def __new__(class_, *args, **kwargs): + if not isinstance(class_._instance, class_): + class_._instance = object.__new__(class_, *args, **kwargs) + return class_._instance + + @abstractmethod + def connect(self) -> None: + """ Establishes a connection to the backend service. """ + pass + + @abstractmethod + def make_request(self, url: str, params=None) -> Union[bytes, str, Dict]: + """ Makes an HTTP request to a given URL using the specified method. """ + pass + + @abstractmethod + def request(self) -> None: + """ Blueprint for the request object. """ + pass + + @abstractmethod + def response(self) -> None: + """ Blueprint for the response object. """ + pass + + @classmethod + def get_instance(cls) -> 'Backend': + """ Returns existing instance, if not then create one. """ + return cls._instance if cls._instance else cls._create_instance() + + @classmethod + def _create_instance(cls) -> 'Backend': + """ Returns the instance after creating it. """ + raise NotImplementedError("Subclasses should implement the creation of instance") + + +class BackendFactory: + def create_backend(self, backend: Optional[Backend] = None) -> Backend: + """ + Create a Backend instance based on Django or manual settings. + + Args: + backend (Optional[Backend], optional): + An optional pre-existing Backend instance. + + Returns: + Backend: A Backend instance. + """ + return backend or self._create_backend_from_settings() + + def _create_backend_from_settings(self) -> Backend: + # TODO: use Django settings to create backend. + pass + + +class Adapter: + """ + Base class for adapters that interact with a backend interface. + + This class should be inherited by adapter classes that facilitate + interaction with different backend implementation. + """ + def __init__(self, backend: Optional[Backend] = None) -> None: + self.backend = BackendFactory().create_backend(backend=backend) + + def request(self): + """ Forward the request to the chosen Backend """ + return self.backend.request() + + def response(self): + """ Forward the response to the chosen Backend """ + return self.backend.response() From 380929608eb2a3020963c1886562de5bfa6387fd Mon Sep 17 00:00:00 2001 From: akash5100 Date: Wed, 13 Sep 2023 18:58:40 +0530 Subject: [PATCH 2/4] Rename module --- .../contentcuration/tests/{appNexus => appnexus}/__init__.py | 0 .../contentcuration/tests/{appNexus => appnexus}/test_base.py | 2 +- .../contentcuration/utils/{appNexus => appnexus}/APILayer.md | 0 .../contentcuration/utils/{appNexus => appnexus}/__init__.py | 0 .../contentcuration/utils/{appNexus => appnexus}/adapters.py | 0 .../contentcuration/utils/{appNexus => appnexus}/backends.py | 0 .../contentcuration/utils/{appNexus => appnexus}/base.py | 0 7 files changed, 1 insertion(+), 1 deletion(-) rename contentcuration/contentcuration/tests/{appNexus => appnexus}/__init__.py (100%) rename contentcuration/contentcuration/tests/{appNexus => appnexus}/test_base.py (93%) rename contentcuration/contentcuration/utils/{appNexus => appnexus}/APILayer.md (100%) rename contentcuration/contentcuration/utils/{appNexus => appnexus}/__init__.py (100%) rename contentcuration/contentcuration/utils/{appNexus => appnexus}/adapters.py (100%) rename contentcuration/contentcuration/utils/{appNexus => appnexus}/backends.py (100%) rename contentcuration/contentcuration/utils/{appNexus => appnexus}/base.py (100%) diff --git a/contentcuration/contentcuration/tests/appNexus/__init__.py b/contentcuration/contentcuration/tests/appnexus/__init__.py similarity index 100% rename from contentcuration/contentcuration/tests/appNexus/__init__.py rename to contentcuration/contentcuration/tests/appnexus/__init__.py diff --git a/contentcuration/contentcuration/tests/appNexus/test_base.py b/contentcuration/contentcuration/tests/appnexus/test_base.py similarity index 93% rename from contentcuration/contentcuration/tests/appNexus/test_base.py rename to contentcuration/contentcuration/tests/appnexus/test_base.py index 0ae614dea4..62b65fd493 100644 --- a/contentcuration/contentcuration/tests/appNexus/test_base.py +++ b/contentcuration/contentcuration/tests/appnexus/test_base.py @@ -1,4 +1,4 @@ -from contentcuration.utils.appNexus.base import Backend, Adapter +from contentcuration.utils.appnexus.base import Backend, Adapter class MockBackend(Backend): _instance = None diff --git a/contentcuration/contentcuration/utils/appNexus/APILayer.md b/contentcuration/contentcuration/utils/appnexus/APILayer.md similarity index 100% rename from contentcuration/contentcuration/utils/appNexus/APILayer.md rename to contentcuration/contentcuration/utils/appnexus/APILayer.md diff --git a/contentcuration/contentcuration/utils/appNexus/__init__.py b/contentcuration/contentcuration/utils/appnexus/__init__.py similarity index 100% rename from contentcuration/contentcuration/utils/appNexus/__init__.py rename to contentcuration/contentcuration/utils/appnexus/__init__.py diff --git a/contentcuration/contentcuration/utils/appNexus/adapters.py b/contentcuration/contentcuration/utils/appnexus/adapters.py similarity index 100% rename from contentcuration/contentcuration/utils/appNexus/adapters.py rename to contentcuration/contentcuration/utils/appnexus/adapters.py diff --git a/contentcuration/contentcuration/utils/appNexus/backends.py b/contentcuration/contentcuration/utils/appnexus/backends.py similarity index 100% rename from contentcuration/contentcuration/utils/appNexus/backends.py rename to contentcuration/contentcuration/utils/appnexus/backends.py diff --git a/contentcuration/contentcuration/utils/appNexus/base.py b/contentcuration/contentcuration/utils/appnexus/base.py similarity index 100% rename from contentcuration/contentcuration/utils/appNexus/base.py rename to contentcuration/contentcuration/utils/appnexus/base.py From cc48e869d6574dfeb1c2d81c306b368fc6e7fa16 Mon Sep 17 00:00:00 2001 From: akash5100 Date: Tue, 19 Sep 2023 00:50:06 +0530 Subject: [PATCH 3/4] the adapter class definitely needs a backend --- .../contentcuration/tests/appnexus/test_base.py | 4 ++-- contentcuration/contentcuration/utils/appnexus/base.py | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/contentcuration/contentcuration/tests/appnexus/test_base.py b/contentcuration/contentcuration/tests/appnexus/test_base.py index 62b65fd493..9bf6e44711 100644 --- a/contentcuration/contentcuration/tests/appnexus/test_base.py +++ b/contentcuration/contentcuration/tests/appnexus/test_base.py @@ -27,10 +27,10 @@ def mockoperation(self): def test_backend_singleton(): b1, b2 = MockBackend.get_instance(), MockBackend.get_instance() - assert id(b1) == id(b2) + assert id(b1) == id(b2) def test_adapter_creation(): - a = MockAdapter() + a = MockAdapter(backend=MockBackend) assert isinstance(a, Adapter) def test_adapter_backend_default(): diff --git a/contentcuration/contentcuration/utils/appnexus/base.py b/contentcuration/contentcuration/utils/appnexus/base.py index f42a13aa77..24e2a0c780 100644 --- a/contentcuration/contentcuration/utils/appnexus/base.py +++ b/contentcuration/contentcuration/utils/appnexus/base.py @@ -47,13 +47,9 @@ class BackendFactory: def create_backend(self, backend: Optional[Backend] = None) -> Backend: """ Create a Backend instance based on Django or manual settings. + :param: backend (optional): An optional pre-existing Backend instance. - Args: - backend (Optional[Backend], optional): - An optional pre-existing Backend instance. - - Returns: - Backend: A Backend instance. + :returns: Backend: A backend instance. """ return backend or self._create_backend_from_settings() @@ -69,7 +65,7 @@ class Adapter: This class should be inherited by adapter classes that facilitate interaction with different backend implementation. """ - def __init__(self, backend: Optional[Backend] = None) -> None: + def __init__(self, backend: Backend) -> None: self.backend = BackendFactory().create_backend(backend=backend) def request(self): From 2905dff355d69bac3c1be6ac089d7b4640c4186c Mon Sep 17 00:00:00 2001 From: akash5100 Date: Thu, 21 Sep 2023 22:08:01 +0530 Subject: [PATCH 4/4] makes backendFactory abstract class --- .../tests/appnexus/test_base.py | 2 -- .../contentcuration/utils/appnexus/base.py | 22 ++++++------------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/contentcuration/contentcuration/tests/appnexus/test_base.py b/contentcuration/contentcuration/tests/appnexus/test_base.py index 9bf6e44711..e8dd657660 100644 --- a/contentcuration/contentcuration/tests/appnexus/test_base.py +++ b/contentcuration/contentcuration/tests/appnexus/test_base.py @@ -1,8 +1,6 @@ from contentcuration.utils.appnexus.base import Backend, Adapter class MockBackend(Backend): - _instance = None - def connect(self) -> None: return super().connect() diff --git a/contentcuration/contentcuration/utils/appnexus/base.py b/contentcuration/contentcuration/utils/appnexus/base.py index 24e2a0c780..f997ed6df2 100644 --- a/contentcuration/contentcuration/utils/appnexus/base.py +++ b/contentcuration/contentcuration/utils/appnexus/base.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from builtins import NotImplementedError -from typing import Optional, Union, Dict +from typing import Union, Dict class Backend(ABC): @@ -43,18 +43,10 @@ def _create_instance(cls) -> 'Backend': raise NotImplementedError("Subclasses should implement the creation of instance") -class BackendFactory: - def create_backend(self, backend: Optional[Backend] = None) -> Backend: - """ - Create a Backend instance based on Django or manual settings. - :param: backend (optional): An optional pre-existing Backend instance. - - :returns: Backend: A backend instance. - """ - return backend or self._create_backend_from_settings() - - def _create_backend_from_settings(self) -> Backend: - # TODO: use Django settings to create backend. +class BackendFactory(ABC): + @abstractmethod + def create_backend(self) -> Backend: + """ Create a Backend instance from the given backend. """ pass @@ -63,10 +55,10 @@ class Adapter: Base class for adapters that interact with a backend interface. This class should be inherited by adapter classes that facilitate - interaction with different backend implementation. + interaction with different backend implementations. """ def __init__(self, backend: Backend) -> None: - self.backend = BackendFactory().create_backend(backend=backend) + self.backend = backend def request(self): """ Forward the request to the chosen Backend """