Merge pull request #1122 from xcube-dev/forman-1121-dataset_descriptions

Data descriptions
xcube-dev · Feb 10, 2025 · b2e8d2c · b2e8d2c
2 parents a2199cc + 44ec1d9
commit b2e8d2c
Show file tree

Hide file tree

Showing 13 changed files with 294 additions and 98 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,19 @@
 ## Changes in 1.8.3 (in development)
 
+### Enhancements
+
+* xcube Server now can be configured to provide abstracts/descriptions for datasets 
+  so they can be rendered as markdown in xcube Viewer 
+  (https://github.com/xcube-dev/xcube-viewer/issues/454). (#1122)
+
+  1. New `description` properties have been added to responses from xcube Server for 
+     datasets and variables.
+  2. User can now provide abstracts or descriptions using markdown format for dataset 
+     configurations in xcube Server. A new configuration setting `Description` 
+     now accompanies settings such as `Title`.
+  3. Default values for the `Description` setting are derived from metadata of 
+     datasets and variable CF attributes.
+
 ### Other changes
 
 * Reformatted code base according to the default settings used by 

diff --git a/examples/serve/panels-demo/config.yaml b/examples/serve/panels-demo/config.yaml
@@ -19,6 +19,22 @@ DataStores:
       - Path: openSR_nordfriesland_S2L2A_selected_dates_10m-v4.levels
         Identifier: waddensea
         Title: Wadden Sea Nordfriesland
+        Description: >-
+          The [**Wadden Sea**](https://en.wikipedia.org/wiki/Wadden_Sea) 
+          (Dutch: _Waddenzee_ [ˈʋɑdə(n)zeː]; 
+          German: _Wattenmeer_ [ˈvatn̩ˌmeːɐ̯];
+          Low German: _Wattensee_ or _Waddenzee_; 
+          Danish: _Vadehavet_;
+          West Frisian: _Waadsee_; 
+          North Frisian: _di Heef_) 
+          is an [intertidal zone](https://en.wikipedia.org/wiki/Intertidal_zone) 
+          in the southeastern part of the 
+          [North Sea](https://en.wikipedia.org/wiki/North_Sea). 
+          It lies between the coast of northwestern continental Europe and the 
+          range of low-lying Frisian Islands, forming a shallow body of water 
+          with tidal flats and wetlands. It has a high biological diversity 
+          and is an important area for both breeding and migrating birds.
+          ![Wadden Sea](https://upload.wikimedia.org/wikipedia/commons/e/e7/13-09-29-nordfriesisches-wattenmeer-RalfR-19.jpg)
         Style: waddensea
         Augmentation:
           Path: compute_indexes.py

diff --git a/test/webapi/datasets/test_context.py b/test/webapi/datasets/test_context.py
@@ -107,7 +107,7 @@ def test_get_dataset_configs_from_stores(self):
 
     def test_get_dataset_configs_with_duplicate_ids_from_stores(self):
         with self.assertRaises(ApiError.InvalidServerConfig) as sce:
-            ctx = get_datasets_ctx("config-datastores-double-ids.yml")
+            get_datasets_ctx("config-datastores-double-ids.yml")
         self.assertEqual(
             "HTTP status 580:"
             " User-defined identifiers can only be assigned to "

diff --git a/test/webapi/datasets/test_controllers.py b/test/webapi/datasets/test_controllers.py
@@ -2,12 +2,13 @@
 # Permissions are hereby granted under the terms of the MIT License:
 # https://opensource.org/licenses/MIT.
 
-
 import os.path
 import unittest
 from test.webapi.helpers import get_api_ctx
 from typing import Any, Optional
 
+import xarray as xr
+
 from xcube.core.new import new_cube
 from xcube.server.api import ApiError
 from xcube.webapi.datasets.context import DatasetsContext
@@ -16,9 +17,11 @@
     find_dataset_places,
     get_color_bars,
     get_dataset,
+    get_dataset_title_and_description,
     get_datasets,
     get_legend,
     get_time_chunk_size,
+    get_variable_title_and_description,
 )
 
 
@@ -193,6 +196,91 @@ def assertPlaceGroupOk(self, feature_collection, expected_count, expected_ids):
         actual_ids = {f["id"] for f in features if "id" in f}
         self.assertEqual(expected_ids, actual_ids)
 
+    def test_dataset_title_and_description(self):
+        dataset = xr.Dataset(
+            attrs={
+                "title": "From title Attr",
+                "name": "From name Attr",
+                "description": "From description Attr",
+                "abstract": "From abstract Attr",
+                "comment": "From comment Attr",
+            }
+        )
+
+        self.assertEqual(
+            ("From Title Conf", "From Description Conf"),
+            get_dataset_title_and_description(
+                dataset,
+                {"Title": "From Title Conf", "Description": "From Description Conf"},
+            ),
+        )
+
+        self.assertEqual(
+            ("From title Attr", "From description Attr"),
+            get_dataset_title_and_description(dataset),
+        )
+
+        del dataset.attrs["title"]
+        del dataset.attrs["description"]
+        self.assertEqual(
+            ("From name Attr", "From abstract Attr"),
+            get_dataset_title_and_description(dataset),
+        )
+
+        del dataset.attrs["name"]
+        del dataset.attrs["abstract"]
+        self.assertEqual(
+            ("", "From comment Attr"),
+            get_dataset_title_and_description(dataset),
+        )
+
+        del dataset.attrs["comment"]
+        self.assertEqual(
+            ("", None),
+            get_dataset_title_and_description(dataset),
+        )
+
+        self.assertEqual(
+            ("From Identifier Conf", None),
+            get_dataset_title_and_description(
+                xr.Dataset(), {"Identifier": "From Identifier Conf"}
+            ),
+        )
+
+    def test_variable_title_and_description(self):
+        variable = xr.DataArray(
+            attrs={
+                "title": "From title Attr",
+                "name": "From name Attr",
+                "long_name": "From long_name Attr",
+                "description": "From description Attr",
+                "abstract": "From abstract Attr",
+                "comment": "From comment Attr",
+            }
+        )
+        self.assertEqual(
+            ("From title Attr", "From description Attr"),
+            get_variable_title_and_description("x", variable),
+        )
+
+        del variable.attrs["title"]
+        del variable.attrs["description"]
+        self.assertEqual(
+            ("From name Attr", "From abstract Attr"),
+            get_variable_title_and_description("x", variable),
+        )
+
+        del variable.attrs["name"]
+        del variable.attrs["abstract"]
+        self.assertEqual(
+            ("From long_name Attr", "From comment Attr"),
+            get_variable_title_and_description("x", variable),
+        )
+
+        del variable.attrs["long_name"]
+        del variable.attrs["comment"]
+        self.assertEqual(("x", None), get_variable_title_and_description("x", variable))
+
 
 class DatasetsAuthControllerTest(DatasetsControllerTestBase):
     @staticmethod
@@ -346,33 +434,21 @@ def test_authorized_access_with_joker_scopes(self):
         granted_scopes = {"read:dataset:*", "read:variable:*"}
         response = get_datasets(ctx, granted_scopes=granted_scopes)
         datasets = self.assertDatasetsOk(response)
-        dataset_ids_dict = {ds["id"]: ds for ds in datasets}
         self.assertEqual(
             {
                 "local_base_1w",
                 "local_base_id",
                 "remote_base_1w",
                 "remote~OLCI-SNS-RAW-CUBE-2.zarr",
             },
-            set(dataset_ids_dict),
-        )
-        dataset_titles_dict = {ds["title"]: ds for ds in datasets}
-        self.assertEqual(
-            {
-                "local_base_1w",
-                "A local base dataset",
-                "remote_base_1w",
-                "A remote base dataset",
-            },
-            set(dataset_titles_dict),
+            {ds["id"] for ds in datasets},
         )
 
     def test_authorized_access_with_specific_scopes(self):
         ctx = get_datasets_ctx(self.get_config())
         granted_scopes = {"read:dataset:remote*", "read:variable:*"}
         response = get_datasets(ctx, granted_scopes=granted_scopes)
         datasets = self.assertDatasetsOk(response)
-        dataset_ids_dict = {ds["id"]: ds for ds in datasets}
         self.assertEqual(
             {
                 # Not selected, because they are substitutes
@@ -381,18 +457,7 @@ def test_authorized_access_with_specific_scopes(self):
                 "remote_base_1w",
                 "remote~OLCI-SNS-RAW-CUBE-2.zarr",
             },
-            set(dataset_ids_dict),
-        )
-        dataset_titles_dict = {ds["title"]: ds for ds in datasets}
-        self.assertEqual(
-            {
-                # Not selected, because they are substitutes
-                # 'local_base_1w',
-                # 'A local base dataset',
-                "remote_base_1w",
-                "A remote base dataset",
-            },
-            set(dataset_titles_dict),
+            {ds["id"] for ds in datasets},
         )
 
 

diff --git a/test/webapi/ows/stac/test_controllers.py b/test/webapi/ows/stac/test_controllers.py
@@ -180,6 +180,9 @@ def read_json(self, filename):
             content = json.load(fp)
         return content
 
+    def setUp(self):
+        self.maxDiff = None
+
     # Commented out to keep coverage checkers happy.
     # @staticmethod
     # def write_json(filename, content):

diff --git a/test/webapi/places/test_routes.py b/test/webapi/places/test_routes.py
@@ -24,6 +24,7 @@ def test_places(self):
                         "sourceEncoding": "utf-8",
                         "sourcePaths": [],
                         "title": "Points inside the cube",
+                        "description": None,
                         "type": "FeatureCollection",
                     },
                     {
@@ -34,6 +35,7 @@ def test_places(self):
                         "sourceEncoding": "utf-8",
                         "sourcePaths": [],
                         "title": "Points outside the cube",
+                        "description": None,
                         "type": "FeatureCollection",
                     },
                 ]

diff --git a/test/webapi/viewer/test_viewer.py b/test/webapi/viewer/test_viewer.py
@@ -193,14 +193,21 @@ def test_add_and_remove_dataset(self):
             new_cube(variables={"analysed_sst": 282.0}),
             ds_id="my_sst_2",
             title="My SST 2",
+            description="It is test 2",
         )
         self.assertEqual("my_sst_2", ds_id_2)
 
         ds_config_1 = self.viewer.datasets_ctx.get_dataset_config(ds_id_1)
-        self.assertEqual({"Identifier": ds_id_1, "Title": "My SST 1"}, ds_config_1)
+        self.assertEqual(
+            {"Identifier": ds_id_1},
+            ds_config_1,
+        )
 
         ds_config_2 = self.viewer.datasets_ctx.get_dataset_config(ds_id_2)
-        self.assertEqual({"Identifier": ds_id_2, "Title": "My SST 2"}, ds_config_2)
+        self.assertEqual(
+            {"Identifier": ds_id_2, "Title": "My SST 2", "Description": "It is test 2"},
+            ds_config_2,
+        )
 
         self.viewer.remove_dataset(ds_id_1)
         with pytest.raises(ApiError.NotFound):
@@ -218,23 +225,34 @@ def test_add_dataset_with_slash_path(self):
             new_cube(variables={"analysed_sst": 280.0}),
             ds_id="mybucket/mysst.levels",
         )
-        ds_id = viewer.add_dataset(ml_ds, title="My SST")
+        ds_id = viewer.add_dataset(ml_ds, title="My SST", description="Test!")
 
         self.assertEqual("mybucket-mysst.levels", ds_id)
 
         ds_config = self.viewer.datasets_ctx.get_dataset_config(ds_id)
-        self.assertEqual({"Identifier": ds_id, "Title": "My SST"}, ds_config)
+        self.assertEqual(
+            {"Identifier": ds_id, "Title": "My SST", "Description": "Test!"}, ds_config
+        )
 
     def test_add_dataset_with_style(self):
         viewer = self.get_viewer(STYLES_CONFIG)
 
         ds_id = viewer.add_dataset(
-            new_cube(variables={"analysed_sst": 280.0}), title="My SST", style="SST"
+            new_cube(variables={"analysed_sst": 280.0}),
+            title="My SST",
+            description="Better use SST",
+            style="SST",
         )
 
         ds_config = self.viewer.datasets_ctx.get_dataset_config(ds_id)
         self.assertEqual(
-            {"Identifier": ds_id, "Title": "My SST", "Style": "SST"}, ds_config
+            {
+                "Identifier": ds_id,
+                "Title": "My SST",
+                "Description": "Better use SST",
+                "Style": "SST",
+            },
+            ds_config,
         )
 
     def test_add_dataset_with_color_mapping(self):
@@ -250,5 +268,10 @@ def test_add_dataset_with_color_mapping(self):
 
         ds_config = self.viewer.datasets_ctx.get_dataset_config(ds_id)
         self.assertEqual(
-            {"Identifier": ds_id, "Title": "My SST", "Style": ds_id}, ds_config
+            {
+                "Identifier": ds_id,
+                "Title": "My SST",
+                "Style": ds_id,
+            },
+            ds_config,
         )
diff --git a/xcube/webapi/datasets/config.py b/xcube/webapi/datasets/config.py
@@ -64,6 +64,7 @@
 
 COMMON_DATASET_PROPERTIES = dict(
     Title=STRING_SCHEMA,
+    Description=STRING_SCHEMA,
     GroupTitle=STRING_SCHEMA,
     Tags=JsonArraySchema(items=STRING_SCHEMA),
     Variables=VARIABLES_SCHEMA,