Handle NetCDF variable length strings (and other VLen types)

lib/iris/fileformats/netcdf/loader.py

@@ -215,10 +215,41 @@ def _get_cf_var_data(cf_var, filename):
         # See https://github.com/SciTools/iris/issues/4994 "Xarray bridge".
         result = cf_var._data_array
     else:
-        total_bytes = cf_var.size * cf_var.dtype.itemsize
+        # Determine size of data; however can't do this for variable length (VLEN)
+        # arrays as the size of the array can only be known by reading the data.
+        # "Variable length" netCDF types have a datatype of `nc.VLType`.
+        if isinstance(
+            getattr(cf_var, "datatype", None), _thread_safe_nc.netCDF4.VLType
+        ):  # TODO(ChrisB): I am accessing netCDF4 directly here - is this ok? Just for type comparison.
+            # We can't know the size of VLen data without reading the variable from disk
+            # first; see https://github.com/Unidata/netcdf-c/issues/1893
+            msg = (
+                f"netCDF variable `{cf_var.cf_name}` is a variable length type of kind {cf_var.dtype} "
+                "and the total data size cannot be known in advance. This may affect the lazy loading "
+                "of the data."
+            )
+            warnings.warn(msg, category=iris.warnings.IrisLoadWarning)
+
+            # A stab in the dark at the mean size of the variable length arrays (this could be anything):
+            mean_vl_array_len = 10
+
+            # Give user the chance to pass a hint of the average variable length array size via
+            # the chunk control context manager. This allows for better decisions to be made on
+            # whether the data should be lazy-loaded or not.
+            if CHUNK_CONTROL.mode is not CHUNK_CONTROL.Modes.AS_DASK:
+                if chunks := CHUNK_CONTROL.var_dim_chunksizes.get(cf_var.cf_name):
+                    if vl_chunk_hint := chunks.get("_vl_hint"):
+                        mean_vl_array_len = vl_chunk_hint
+
+            # In this case, cf_var.size will just return the known dimension size.
+            # Special handling for strings (`str` type) as these don't have an itemsize attribute; assume 4 bytes
+            itemsize = 4 if cf_var.dtype is str else cf_var.dtype.itemsize
+            total_bytes = cf_var.size * mean_vl_array_len * itemsize
+        else:
+            # Normal NCVariable type:
+            total_bytes = cf_var.size * cf_var.dtype.itemsize
+
         if total_bytes < _LAZYVAR_MIN_BYTES:
-            # Don't make a lazy array, as it will cost more memory AND more time to access.
-            # Instead fetch the data immediately, as a real array, and return that.
             result = cf_var[:]
 
         else:
@@ -228,10 +259,13 @@ def _get_cf_var_data(cf_var, filename):
             dtype = _get_actual_dtype(cf_var)
 
             # Make a data-proxy that mimics array access and can fetch from the file.
+            # Note: Special handling needed for "variable length string" types which
+            # return a dtype of `str`, rather than a numpy type; use `S1` in this case.
+            fill_dtype = "S1" if cf_var.dtype is str else cf_var.dtype.str[1:]
             fill_value = getattr(
                 cf_var.cf_data,
                 "_FillValue",
-                _thread_safe_nc.default_fillvals[cf_var.dtype.str[1:]],
+                _thread_safe_nc.default_fillvals[fill_dtype],
             )
             proxy = NetCDFDataProxy(
                 cf_var.shape, dtype, filename, cf_var.cf_name, fill_value

lib/iris/tests/unit/fileformats/netcdf/loader/test__get_cf_var_data.py

@@ -11,6 +11,7 @@
 from unittest import mock
 
 import dask.array as da
+from netCDF4 import VLType
 import numpy as np
 
 from iris._lazy_data import _optimum_chunksize
@@ -33,7 +34,8 @@ def _make(self, chunksizes=None, shape=None, dtype="i4", **extra_properties):
         cf_data.chunking = mock.MagicMock(return_value=chunksizes)
         if shape is None:
             shape = self.shape
-        dtype = np.dtype(dtype)
+        if dtype is not str:  # for testing VLen str arrays (dtype=`class <str>`)
+            dtype = np.dtype(dtype)
         cf_var = mock.MagicMock(
             spec=iris.fileformats.cf.CFVariable,
             dtype=dtype,
@@ -103,6 +105,51 @@ def test_arraytype__100f8_is_real(self):
         var_data = _get_cf_var_data(cf_var, self.filename)
         self.assertIs(var_data, mock.sentinel.real_data_accessed)
 
+    def test_vltype__1000str_is_lazy(self):
+        # Variable length string type
+        mock_vltype = mock.Mock(spec=VLType, dtype=str, name="varlen string type")
+        cf_var = self._make(shape=(1000,), dtype=str, datatype=mock_vltype)
+        var_data = _get_cf_var_data(cf_var, self.filename)
+        self.assertIsInstance(var_data, da.Array)
+
+    def test_vltype__1000str_is_real_with_hint(self):
+        # Variable length string type with a hint on the array variable length size
+        mock_vltype = mock.Mock(spec=VLType, dtype=str, name="varlen string type")
+        cf_var = self._make(shape=(100,), dtype=str, datatype=mock_vltype)
+        with CHUNK_CONTROL.set("DUMMY_VAR", _vl_hint=1):
+            var_data = _get_cf_var_data(cf_var, self.filename)
+        self.assertIs(var_data, mock.sentinel.real_data_accessed)
+
+    def test_vltype__100str_is_real(self):
+        # Variable length string type
+        mock_vltype = mock.Mock(spec=VLType, dtype=str, name="varlen string type")
+        cf_var = self._make(shape=(100,), dtype=str, datatype=mock_vltype)
+        var_data = _get_cf_var_data(cf_var, self.filename)
+        self.assertIs(var_data, mock.sentinel.real_data_accessed)
+
+    def test_vltype__100str_is_lazy_with_hint(self):
+        # Variable length string type with a hint on the array variable length size
+        mock_vltype = mock.Mock(spec=VLType, dtype=str, name="varlen string type")
+        cf_var = self._make(shape=(100,), dtype=str, datatype=mock_vltype)
+        with CHUNK_CONTROL.set("DUMMY_VAR", _vl_hint=50):
+            var_data = _get_cf_var_data(cf_var, self.filename)
+        self.assertIsInstance(var_data, da.Array)
+
+    def test_vltype__100f8_is_lazy(self):
+        # Variable length float64 type
+        mock_vltype = mock.Mock(spec=VLType, dtype="f8", name="varlen float64 type")
+        cf_var = self._make(shape=(1000,), dtype="f8", datatype=mock_vltype)
+        var_data = _get_cf_var_data(cf_var, self.filename)
+        self.assertIsInstance(var_data, da.Array)
+
+    def test_vltype__100f8_is_real_with_hint(self):
+        # Variable length float64 type with a hint on the array variable length size
+        mock_vltype = mock.Mock(spec=VLType, dtype="f8", name="varlen float64 type")
+        cf_var = self._make(shape=(100,), dtype="f8", datatype=mock_vltype)
+        with CHUNK_CONTROL.set("DUMMY_VAR", _vl_hint=2):
+            var_data = _get_cf_var_data(cf_var, self.filename)
+        self.assertIs(var_data, mock.sentinel.real_data_accessed)
+
     def test_cf_data_emulation(self):
         # Check that a variable emulation object passes its real data directly.
         emulated_data = mock.Mock()