From 392608d16f3437b54d8cbf7f726a18d4a8f76e77 Mon Sep 17 00:00:00 2001 From: Mattia Almansi Date: Fri, 6 Sep 2024 13:02:42 +0200 Subject: [PATCH 1/4] Allow cads-adaptors to directly write on the cache filesystem --- cads_adaptors/adaptors/__init__.py | 10 ++++++++-- tests/test_10_cache_tmp_path.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 tests/test_10_cache_tmp_path.py diff --git a/cads_adaptors/adaptors/__init__.py b/cads_adaptors/adaptors/__init__.py index af075e08..e66a6417 100644 --- a/cads_adaptors/adaptors/__init__.py +++ b/cads_adaptors/adaptors/__init__.py @@ -1,5 +1,6 @@ import abc import contextlib +import pathlib from typing import Any, BinaryIO import cads_adaptors.tools.logger @@ -85,6 +86,7 @@ def __init__( self, form: list[dict[str, Any]] | dict[str, Any] | None, context: Context | None = None, + cache_tmp_path: pathlib.Path | None = None, **config: Any, ) -> None: self.form = form @@ -93,6 +95,9 @@ def __init__( self.context = Context() else: self.context = context + self.cache_tmp_path = ( + pathlib.Path() if cache_tmp_path is None else cache_tmp_path + ) @abc.abstractmethod def normalise_request(self, request: Request) -> Request: @@ -185,10 +190,11 @@ def retrieve(self, request: Request) -> BinaryIO: time_sleep = 0 time.sleep(time_sleep) - with open("dummy.grib", "wb") as fp: + dummy_file = self.cache_tmp_path / "dummy.grib" + with dummy_file.open("wb") as fp: with open("/dev/urandom", "rb") as random: while size > 0: length = min(size, 10240) fp.write(random.read(length)) size -= length - return open("dummy.grib", "rb") + return dummy_file.open("rb") diff --git a/tests/test_10_cache_tmp_path.py b/tests/test_10_cache_tmp_path.py new file mode 100644 index 00000000..2a32d394 --- /dev/null +++ b/tests/test_10_cache_tmp_path.py @@ -0,0 +1,10 @@ +import pathlib + +import cads_adaptors + + +def test_cache_tmp_path_dummy_adaptor(tmp_path: pathlib.Path) -> None: + dummy_adaptor = cads_adaptors.DummyAdaptor(None, cache_tmp_path=tmp_path) + result = dummy_adaptor.retrieve({"size": 1}) + assert result.name == str(tmp_path / "dummy.grib") + assert (tmp_path / "dummy.grib").stat().st_size == 1 From d5af22c5dba0bb31f4ad9cc985f0afbacb0badb3 Mon Sep 17 00:00:00 2001 From: Mattia Almansi Date: Mon, 9 Sep 2024 09:46:10 +0200 Subject: [PATCH 2/4] add log --- cads_adaptors/adaptors/__init__.py | 7 +++++++ tests/test_10_cache_tmp_path.py | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cads_adaptors/adaptors/__init__.py b/cads_adaptors/adaptors/__init__.py index e66a6417..6e89494b 100644 --- a/cads_adaptors/adaptors/__init__.py +++ b/cads_adaptors/adaptors/__init__.py @@ -189,12 +189,19 @@ def retrieve(self, request: Request) -> BinaryIO: except Exception: time_sleep = 0 + self.context.add_stdout(f"Sleeping {time_sleep} s") time.sleep(time_sleep) + dummy_file = self.cache_tmp_path / "dummy.grib" + self.context.add_stdout(f"Writing {size} B to {dummy_file!s}") + tic = time.perf_counter() with dummy_file.open("wb") as fp: with open("/dev/urandom", "rb") as random: while size > 0: length = min(size, 10240) fp.write(random.read(length)) size -= length + toc = time.perf_counter() + self.context.add_stdout(f"Elapsed time to write {size} B: {toc - tic} s") + return dummy_file.open("rb") diff --git a/tests/test_10_cache_tmp_path.py b/tests/test_10_cache_tmp_path.py index 2a32d394..67a701e0 100644 --- a/tests/test_10_cache_tmp_path.py +++ b/tests/test_10_cache_tmp_path.py @@ -1,3 +1,4 @@ +import os import pathlib import cads_adaptors @@ -7,4 +8,4 @@ def test_cache_tmp_path_dummy_adaptor(tmp_path: pathlib.Path) -> None: dummy_adaptor = cads_adaptors.DummyAdaptor(None, cache_tmp_path=tmp_path) result = dummy_adaptor.retrieve({"size": 1}) assert result.name == str(tmp_path / "dummy.grib") - assert (tmp_path / "dummy.grib").stat().st_size == 1 + assert os.path.getsize(result.name) == 1 From af73886e13c9d93fc743ec69c4995aab5988984a Mon Sep 17 00:00:00 2001 From: Mattia Almansi Date: Mon, 9 Sep 2024 10:12:11 +0200 Subject: [PATCH 3/4] fix log --- cads_adaptors/adaptors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cads_adaptors/adaptors/__init__.py b/cads_adaptors/adaptors/__init__.py index 6e89494b..52093e00 100644 --- a/cads_adaptors/adaptors/__init__.py +++ b/cads_adaptors/adaptors/__init__.py @@ -202,6 +202,6 @@ def retrieve(self, request: Request) -> BinaryIO: fp.write(random.read(length)) size -= length toc = time.perf_counter() - self.context.add_stdout(f"Elapsed time to write {size} B: {toc - tic} s") + self.context.add_stdout(f"Elapsed time to write: {toc - tic} s") return dummy_file.open("rb") From aeeb348050e00b6a0f7e049e34fad69ec8068207 Mon Sep 17 00:00:00 2001 From: Mattia Almansi Date: Mon, 9 Sep 2024 10:19:20 +0200 Subject: [PATCH 4/4] better log --- cads_adaptors/adaptors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cads_adaptors/adaptors/__init__.py b/cads_adaptors/adaptors/__init__.py index 52093e00..c0eec9e5 100644 --- a/cads_adaptors/adaptors/__init__.py +++ b/cads_adaptors/adaptors/__init__.py @@ -202,6 +202,6 @@ def retrieve(self, request: Request) -> BinaryIO: fp.write(random.read(length)) size -= length toc = time.perf_counter() - self.context.add_stdout(f"Elapsed time to write: {toc - tic} s") + self.context.add_stdout(f"Elapsed time to write the file: {toc - tic} s") return dummy_file.open("rb")