Merge pull request #238 from ecmwf-projects/fix-csv-header

Observations: Fix the variables and units in the CSV header

cads_adaptors/adaptors/cadsobs/csv.py

@@ -1,7 +1,6 @@
 import logging
 from pathlib import Path
 
-import dask
 import xarray
 
 from cads_adaptors.adaptors.cadsobs.models import RetrieveArgs
@@ -66,11 +65,14 @@ def get_csv_header(
     time_end = "{:%Y%m%d}".format(
         cdm_lite_dataset.report_timestamp[-1].compute().dt.date
     )
-    vars_and_units = zip(
-        dask.array.unique(cdm_lite_dataset.observed_variable.data)
-        .compute()
-        .astype("U"),
-        dask.array.unique(cdm_lite_dataset.units.data).compute().astype("U"),
+    # Subset the dataset to get variables and units, drop duplicates, encode and convert
+    # to tuples.
+    vars_and_units = list(
+        cdm_lite_dataset[["observed_variable", "units"]]
+        .to_dataframe()
+        .drop_duplicates()
+        .astype("U")
+        .itertuples(index=False, name=None)
     )
     varstr = "\n".join([f"# {v} [{u}]" for v, u in vars_and_units])
     header_params = dict(

tests/test_cadsobs_adaptor.py

@@ -126,7 +126,7 @@ def _send_request(self, endpoint, method, payload):
 TEST_REQUEST = {
     "time_aggregation": "daily",
     "format": "netCDF",
-    "variable": ["maximum_air_temperature"],
+    "variable": ["maximum_air_temperature", "maximum_relative_humidity"],
     "year": ["2007"],
     "month": ["11"],
     "day": [
@@ -185,6 +185,26 @@ def test_adaptor(tmp_path, monkeypatch):
     assert actual.dimensions["index"].size > 0
 
 
+def test_adaptor_csv(tmp_path, monkeypatch):
+    monkeypatch.setattr(
+        "cads_adaptors.adaptors.cadsobs.adaptor.CadsobsApiClient",
+        MockerCadsobsApiClient,
+    )
+    test_form = {}
+
+    adaptor = ObservationsAdaptor(test_form, **TEST_ADAPTOR_CONFIG)
+    test_request_csv = TEST_REQUEST.copy()
+    test_request_csv["format"] = "csv"
+    result = adaptor.retrieve(test_request_csv)
+    tempfile = Path(tmp_path, "test_adaptor.csv")
+    with tempfile.open("wb") as tmpf:
+        tmpf.write(result.read())
+    assert tempfile.stat().st_size > 0
+    file_lines = tempfile.read_text().split("\n")
+    assert "# daily_maximum_air_temperature [K]" in file_lines
+    assert "# daily_maximum_relative_humidity [%]" in file_lines
+
+
 def test_adaptor_error(tmp_path, monkeypatch):
     monkeypatch.setattr(
         "cads_adaptors.adaptors.cadsobs.adaptor.CadsobsApiClient",