Handle NetCDF variable length strings (and other VLen types)

lib/iris/fileformats/netcdf/_thread_safe_nc.py

@@ -20,6 +20,7 @@
 # Doesn't need thread protection, but this allows all netCDF4 refs to be
 #  replaced with thread_safe refs.
 default_fillvals = netCDF4.default_fillvals
+VLType = netCDF4.VLType
 
 
 class _ThreadSafeWrapper(ABC):

lib/iris/fileformats/netcdf/loader.py

@@ -196,6 +196,11 @@ def _get_actual_dtype(cf_var):
 # mostly done for speed improvement.  See https://github.com/SciTools/iris/pull/5069
 _LAZYVAR_MIN_BYTES = 5000
 
+# A stab in the dark at the mean length of the "ragged dimension" for netCDF "variable
+# length arrays" (`NetCDF.VLType` type). Total array size is unknown until the variable is
+# read in. Making this number bigger makes it more likely an array will be loaded lazily.
+_MEAN_VL_ARRAY_LEN = 10
+
 
 def _get_cf_var_data(cf_var, filename):
     """Get an array representing the data of a CF variable.
@@ -215,23 +220,53 @@ def _get_cf_var_data(cf_var, filename):
         # See https://github.com/SciTools/iris/issues/4994 "Xarray bridge".
         result = cf_var._data_array
     else:
-        total_bytes = cf_var.size * cf_var.dtype.itemsize
+        # Determine size of data; however can't do this for variable length (VLEN)
+        # netCDF arrays as the size of the array can only be known by reading the
+        # data; see https://github.com/Unidata/netcdf-c/issues/1893.
+        # Note: "Variable length" netCDF types have a datatype of `nc.VLType`.
+        if isinstance(getattr(cf_var, "datatype", None), _thread_safe_nc.VLType):
+            msg = (
+                f"NetCDF variable `{cf_var.cf_name}` is a variable length type of kind {cf_var.dtype} "
+                "thus the total data size cannot be known in advance. This may affect the lazy loading "
+                "of the data."
+            )
+            warnings.warn(msg, category=iris.warnings.IrisLoadWarning)
+
+            # Give user the chance to pass a hint of the average variable length array size via
+            # the chunk control context manager. This allows for better decisions to be made on
+            # whether the data should be lazy-loaded or not.
+            mean_vl_array_len = _MEAN_VL_ARRAY_LEN
+            if CHUNK_CONTROL.mode is not CHUNK_CONTROL.Modes.AS_DASK:
+                if chunks := CHUNK_CONTROL.var_dim_chunksizes.get(cf_var.cf_name):
+                    if vl_chunk_hint := chunks.get("_vl_hint"):
+                        mean_vl_array_len = vl_chunk_hint
+
+            # Special handling for strings (`str` type) as these don't have an itemsize attribute;
+            # assume 4 bytes which is sufficient for unicode character storage
+            itemsize = 4 if cf_var.dtype is str else cf_var.dtype.itemsize
+
+            # For `VLType` cf_var.size will just return the known dimension size.
+            total_bytes = cf_var.size * mean_vl_array_len * itemsize
+        else:
+            # Normal NCVariable type:
+            total_bytes = cf_var.size * cf_var.dtype.itemsize
+
         if total_bytes < _LAZYVAR_MIN_BYTES:
-            # Don't make a lazy array, as it will cost more memory AND more time to access.
-            # Instead fetch the data immediately, as a real array, and return that.
             result = cf_var[:]
-
         else:
             # Get lazy chunked data out of a cf variable.
             # Creates Dask wrappers around data arrays for any cube components which
             # can have lazy values, e.g. Cube, Coord, CellMeasure, AuxiliaryVariable.
             dtype = _get_actual_dtype(cf_var)
 
             # Make a data-proxy that mimics array access and can fetch from the file.
+            # Note: Special handling needed for "variable length string" types which
+            # return a dtype of `str`, rather than a numpy type; use `S1` in this case.
+            fill_dtype = "S1" if cf_var.dtype is str else cf_var.dtype.str[1:]
             fill_value = getattr(
                 cf_var.cf_data,
                 "_FillValue",
-                _thread_safe_nc.default_fillvals[cf_var.dtype.str[1:]],
+                _thread_safe_nc.default_fillvals[fill_dtype],
             )
             proxy = NetCDFDataProxy(
                 cf_var.shape, dtype, filename, cf_var.cf_name, fill_value

lib/iris/tests/unit/fileformats/netcdf/loader/test__get_cf_var_data.py

@@ -15,6 +15,7 @@
 
 from iris._lazy_data import _optimum_chunksize
 import iris.fileformats.cf
+from iris.fileformats.netcdf._thread_safe_nc import VLType
 from iris.fileformats.netcdf.loader import CHUNK_CONTROL, _get_cf_var_data
 
 
@@ -33,7 +34,8 @@ def _make(self, chunksizes=None, shape=None, dtype="i4", **extra_properties):
         cf_data.chunking = mock.MagicMock(return_value=chunksizes)
         if shape is None:
             shape = self.shape
-        dtype = np.dtype(dtype)
+        if dtype is not str:  # for testing VLen str arrays (dtype=`class <str>`)
+            dtype = np.dtype(dtype)
         cf_var = mock.MagicMock(
             spec=iris.fileformats.cf.CFVariable,
             dtype=dtype,
@@ -103,6 +105,51 @@ def test_arraytype__100f8_is_real(self):
         var_data = _get_cf_var_data(cf_var, self.filename)
         self.assertIs(var_data, mock.sentinel.real_data_accessed)
 
+    def test_vltype__1000str_is_lazy(self):
+        # Variable length string type
+        mock_vltype = mock.Mock(spec=VLType, dtype=str, name="varlen string type")
+        cf_var = self._make(shape=(1000,), dtype=str, datatype=mock_vltype)
+        var_data = _get_cf_var_data(cf_var, self.filename)
+        self.assertIsInstance(var_data, da.Array)
+
+    def test_vltype__1000str_is_real_with_hint(self):
+        # Variable length string type with a hint on the array variable length size
+        mock_vltype = mock.Mock(spec=VLType, dtype=str, name="varlen string type")
+        cf_var = self._make(shape=(100,), dtype=str, datatype=mock_vltype)
+        with CHUNK_CONTROL.set("DUMMY_VAR", _vl_hint=1):
+            var_data = _get_cf_var_data(cf_var, self.filename)
+        self.assertIs(var_data, mock.sentinel.real_data_accessed)
+
+    def test_vltype__100str_is_real(self):
+        # Variable length string type
+        mock_vltype = mock.Mock(spec=VLType, dtype=str, name="varlen string type")
+        cf_var = self._make(shape=(100,), dtype=str, datatype=mock_vltype)
+        var_data = _get_cf_var_data(cf_var, self.filename)
+        self.assertIs(var_data, mock.sentinel.real_data_accessed)
+
+    def test_vltype__100str_is_lazy_with_hint(self):
+        # Variable length string type with a hint on the array variable length size
+        mock_vltype = mock.Mock(spec=VLType, dtype=str, name="varlen string type")
+        cf_var = self._make(shape=(100,), dtype=str, datatype=mock_vltype)
+        with CHUNK_CONTROL.set("DUMMY_VAR", _vl_hint=50):
+            var_data = _get_cf_var_data(cf_var, self.filename)
+        self.assertIsInstance(var_data, da.Array)
+
+    def test_vltype__100f8_is_lazy(self):
+        # Variable length float64 type
+        mock_vltype = mock.Mock(spec=VLType, dtype="f8", name="varlen float64 type")
+        cf_var = self._make(shape=(1000,), dtype="f8", datatype=mock_vltype)
+        var_data = _get_cf_var_data(cf_var, self.filename)
+        self.assertIsInstance(var_data, da.Array)
+
+    def test_vltype__100f8_is_real_with_hint(self):
+        # Variable length float64 type with a hint on the array variable length size
+        mock_vltype = mock.Mock(spec=VLType, dtype="f8", name="varlen float64 type")
+        cf_var = self._make(shape=(100,), dtype="f8", datatype=mock_vltype)
+        with CHUNK_CONTROL.set("DUMMY_VAR", _vl_hint=2):
+            var_data = _get_cf_var_data(cf_var, self.filename)
+        self.assertIs(var_data, mock.sentinel.real_data_accessed)
+
     def test_cf_data_emulation(self):
         # Check that a variable emulation object passes its real data directly.
         emulated_data = mock.Mock()