Refactor to create a more maintainable structure

cads_adaptors/__init__.py

@@ -22,14 +22,13 @@
     # Local copy or not installed with setuptools
     __version__ = "999"
 
-from .adaptor import AbstractAdaptor, DummyAdaptor
-from .adaptor_cds import (
-    AbstractCdsAdaptor,
-    DirectMarsCdsAdaptor,
-    LegacyCdsAdaptor,
-    MarsCdsAdaptor,
-    UrlCdsAdaptor,
-)
+from cads_adaptors.adaptors import AbstractAdaptor, DummyAdaptor
+
+from cads_adaptors.adaptors.cds import AbstractCdsAdaptor
+from cads_adaptors.adaptors.url import UrlCdsAdaptor
+from cads_adaptors.adaptors.mars import MarsCdsAdaptor, DirectMarsCdsAdaptor
+from cads_adaptors.adaptors.legacy import LegacyCdsAdaptor
+
 from .tools.adaptor_tools import get_adaptor_class
 
 __all__ = [

cads_adaptors/adaptor.py → cads_adaptors/adaptors/__init__.py

@@ -1,6 +1,9 @@
 import abc
 from typing import Any, BinaryIO
 
+from cads_adaptors import constraints, costing
+
+
 Request = dict[str, Any]
 
 

cads_adaptors/adaptors/cds.py

@@ -0,0 +1,28 @@
+from typing import Any
+
+from cads_adaptors import constraints, costing
+from cads_adaptors.adaptors import AbstractAdaptor, Request
+
+
+class AbstractCdsAdaptor(AbstractAdaptor):
+    resources = {"CADS_ADAPTORS": 1}
+
+    def __init__(self, form: dict[str, Any], **config: Any):
+        self.form = form
+        self.constraints = config.pop("constraints", [])
+        self.mapping = config.pop("mapping", {})
+        self.licences: list[tuple[str, int]] = config.pop("licences", [])
+        self.config = config
+
+    def validate(self, request: Request) -> bool:
+        return True
+
+    def apply_constraints(self, request: Request) -> dict[str, Any]:
+        return constraints.validate_constraints(self.form, request, self.constraints)
+
+    def estimate_costs(self, request: Request) -> dict[str, int]:
+        costs = {"size": costing.estimate_size(self.form, request, self.constraints)}
+        return costs
+
+    def get_licences(self, request: Request) -> list[tuple[str, int]]:
+        return self.licences

cads_adaptors/adaptors/legacy.py

@@ -0,0 +1,18 @@
+
+from typing import BinaryIO
+
+from cads_adaptors.adaptors import cds, Request
+
+class LegacyCdsAdaptor(cds.AbstractCdsAdaptor):
+    def retrieve(self, request: Request) -> BinaryIO:
+        import cdsapi
+
+        # parse input options
+        collection_id = self.config.pop("collection_id", None)
+        if not collection_id:
+            raise ValueError("collection_id is required in request")
+
+        # retrieve data
+        client = cdsapi.Client(self.config["url"], self.config["key"], retry_max=1)
+        result_path = client.retrieve(collection_id, request).download()
+        return open(result_path, "rb")

cads_adaptors/adaptors/mars.py

@@ -0,0 +1,42 @@
+
+import os
+from typing import BinaryIO
+
+from cads_adaptors import mapping
+from cads_adaptors.adaptors import cds, Request
+
+
+class DirectMarsAdaptor(cds.AbstractCdsAdaptor):
+    resources = {"MARS_CLIENT": 1}
+
+    def retrieve(self, request: Request) -> BinaryIO:
+        import subprocess
+
+        with open("r", "w") as fp:
+            print("retrieve, target=data.grib", file=fp)
+            for key, value in request.items():
+                if not isinstance(value, (list, tuple)):
+                    value = [value]
+                print(f", {key}={'/'.join(str(v) for v in value)}", file=fp)
+
+        env = dict(**os.environ)
+        # FIXME: set with the namespace and user_id
+        namespace = "cads"
+        user_id = 0
+        env["MARS_USER"] = f"{namespace}-{user_id}"
+
+        subprocess.run(["/usr/local/bin/mars", "r"], check=True, env=env)
+
+        return open("data.grib")  # type: ignore
+
+
+class MarsAdaptor(cds.AbstractCdsAdaptor):
+    def retrieve(self, request: Request) -> BinaryIO:
+        format = request.pop("format", ["grib"])
+        assert len(format) == 1
+
+        mapped_request = mapping.apply_mapping(request, self.mapping)  # type: ignore
+        if format[0] != "grib":
+            # FIXME: reformat if needed
+            pass
+        return super().retrieve(mapped_request)

cads_adaptors/adaptors/url.py

@@ -0,0 +1,26 @@
+
+from typing import BinaryIO
+
+from cads_adaptors import mapping
+from cads_adaptors.adaptors import cds, Request
+
+
+class UrlAdaptor(cds.AbstractCdsAdaptor):
+    def retrieve(self, request: Request) -> BinaryIO:
+        from cads_adaptors.tools import url_tools
+
+        data_format = request.pop("format", "zip")
+
+        if data_format not in {"zip", "tgz"}:
+            raise ValueError(f"{data_format=} is not supported")
+
+        mapped_request = mapping.apply_mapping(request, self.mapping)  # type: ignore
+
+        requests_urls = url_tools.requests_to_urls(
+            mapped_request, patterns=self.config["patterns"]
+        )
+
+        path = url_tools.download_from_urls(
+            [ru["url"] for ru in requests_urls], data_format=data_format
+        )
+        return open(path, "rb")

cads_adaptors/tools/adaptor_tools.py

@@ -1,11 +1,11 @@
 from typing import Any
 
-from .. import adaptor
+from cads_adaptors.adaptors import AbstractAdaptor
 
 
 def get_adaptor_class(
     entry_point: str, setup_code: str | None = None
-) -> type[adaptor.AbstractAdaptor]:
+) -> type[AbstractAdaptor]:
     from cacholote import decode
 
     try:
@@ -17,8 +17,8 @@ def get_adaptor_class(
             raise TypeError
         exec(setup_code)
         adaptor_class = eval(entry_point)
-    if not issubclass(adaptor_class, adaptor.AbstractAdaptor):
-        raise TypeError(f"{adaptor_class!r} is not subclass of AbstractAdaptor")
+    if not issubclass(adaptor_class, AbstractAdaptor):
+        raise TypeError(f"{adaptor_class!r} is not subclass of Base")
     return adaptor_class  # type: ignore