Full test coverage for GRIB1 (#583)

* More integration tests for GRIB1 files.

* Test coverage for timeRangeIndicators.

* Add Polar Stereo South coverage for GRIB1.

* Add Reduced GG coverage for GRIB 1.

* Add coverage for identification of y-wind in GRIB1.

* Add coverage for CF-mapping in GRIB1.

* Widen reduced GG GRIB1 test to cover ML level type.

* Refactor test_reduced_gg_grib1 into TestBasicLoad.

* Tidy new tests.

src/iris_grib/tests/integration/load_convert/test_sample_file_loads.py

@@ -10,11 +10,18 @@
 
 """
 
+from __future__ import annotations
+
 # import iris_grib.tests first so that some things can be initialised
 # before importing anything else.
 import iris_grib.tests as tests
 
+from pathlib import Path
+
+import eccodes
 import iris
+import pytest
+
 
 _RESULTDIR_PREFIX = ("integration", "load_convert", "sample_file_loads")
 
@@ -111,12 +118,33 @@ def test_reduced_ll(self):
         )
         self.assertCML(cube, _RESULTDIR_PREFIX + ("reduced_ll_grib1.cml",))
 
-    def test_reduced_gg(self):
+    def test_reduced_gg_grib1(self):
+        cube = iris.load_cube(
+            Path(eccodes.codes_samples_path()) / "reduced_gg_ml_grib1.tmpl"
+        )
+        self.assertCML(cube, _RESULTDIR_PREFIX + ("reduced_gg_grib1.cml",))
+
+    def test_reduced_gg_grib2(self):
         cube = iris.load_cube(
             tests.get_data_path(("GRIB", "reduced", "reduced_gg.grib2"))
         )
         self.assertCML(cube, _RESULTDIR_PREFIX + ("reduced_gg_grib2.cml",))
 
+    def test_second_order_packing(self):
+        cube = iris.load_cube(
+            tests.get_data_path(
+                ("GRIB", "grib1_second_order_packing", "GRIB_00008_FRANX01")
+            )
+        )
+        self.assertCML(cube, _RESULTDIR_PREFIX + ("second_order_packing.cml",))
+
+    def test_bulletin_headers(self):
+        for byte_len in (40, 41):
+            cube = iris.load_cube(
+                tests.get_data_path(("GRIB", "bulletin", f"{byte_len}bytes.grib"))
+            )
+            self.assertCML(cube, _RESULTDIR_PREFIX + (f"bulletin_{byte_len}bytes.cml",))
+
 
 @tests.skip_data
 class TestIjDirections(tests.IrisGribTest):
@@ -197,5 +225,95 @@ def test_shape_of_earth_grib1(self):
         self.assertCML(cube, _RESULTDIR_PREFIX + ("earth_shape_grib1.cml",))
 
 
+class TestTimesGrib1:
+    # Our codebase has support for many timeRangeIndicator values in GRIB1
+    #  files, but we do not have files demonstrating these. This class
+    #  generates appropriate GRIB1 files so that we can fully refactor the
+    #  loading code while ensuring continued support for all possible
+    #  scenarios.
+
+    @pytest.fixture(params=[1, 2, 3, 4, 5, 10, 113, 118, 123, 124], autouse=True)
+    # Note that the following values are not supported by Eccodes - it can
+    #  not provide a startStep value unless the timeRangeIndicator is present
+    #  in eccodes/definitions/grib1/localConcepts/edzw/stepType.def:
+    #  [51, 114, 115, 116, 117, 125].
+    def _get_time_range_file(self, request, tmp_path):
+        save_file = tmp_path / "TestTimes.grib1"
+        time_range_indicator = request.param
+
+        # Make a file with 10 ascending time steps.
+        with save_file.open("wb") as open_file:
+            for time_step in range(10):
+                grib_message = eccodes.codes_grib_new_from_samples("GRIB1")
+                eccodes.codes_set_long(grib_message, "P1", time_step)
+                eccodes.codes_set_long(grib_message, "P2", time_step + 1)
+                eccodes.codes_set_long(
+                    grib_message, "timeRangeIndicator", time_range_indicator
+                )
+                eccodes.codes_write(grib_message, open_file)
+                eccodes.codes_release(grib_message)
+
+        self.time_range_indicator = time_range_indicator
+        self.save_file = save_file
+
+    def test_time_range(self):
+        cube = iris.load_cube(self.save_file)
+        tests.IrisGribTest().assertCML(
+            cube, _RESULTDIR_PREFIX + (f"time_range_{self.time_range_indicator}.cml",)
+        )
+
+
+class TestFullCoverageGrib1:
+    # We do not have a full set of GRIB1 files that exercise our entire GRIB1
+    #  loading code. This class generates files to cover those gaps. We use CML
+    #  testing since, in the absence of bug reports after years of use (time
+    #  of writing: 2024), we must assume that current functionality is
+    #  satisfactory. Integration tests of this sort will allow us to fully
+    #  refactor GRIB1 loading without any change in user experience.
+
+    id_path_codes = [
+        (
+            "polar_stereo_south",
+            tests.get_data_path(("GRIB", "polar_stereo", "ST4.2013052210.01h")),
+            [("projectionCentreFlag", 1)],
+        ),
+        (
+            "y_wind",
+            tests.get_data_path(("GRIB", "gaussian", "regular_gg.grib1")),
+            [("indicatorOfParameter", 34)],
+        ),
+        (
+            "mapped_cf_data",
+            Path(eccodes.codes_samples_path()) / "GRIB1.tmpl",
+            [("table2Version", 128), ("centre", 98), ("indicatorOfParameter", 34)],
+        ),
+    ]
+
+    @pytest.fixture(
+        params=id_path_codes,
+        ids=[id for id, path, codes in id_path_codes],
+        autouse=True,
+    )
+    def _get_grib1_file(self, request, tmp_path):
+        id_, path, codes = request.param
+        path_original = Path(path)
+        path_modified = tmp_path / "tmp_file.grib1"
+        with path_original.open("rb") as file_original:
+            gid = eccodes.codes_grib_new_from_file(file_original)
+            for key, value in codes:
+                eccodes.codes_set(gid, key, value)
+            with path_modified.open("wb") as file_modified:
+                eccodes.codes_write(gid, file_modified)
+            eccodes.codes_release(gid)
+        self.id_ = id_
+        self.file_path = path_modified
+
+    def test_grib1(self):
+        cube = iris.load_cube(self.file_path)
+        tests.IrisGribTest().assertCML(
+            cube, _RESULTDIR_PREFIX + (f"{self.id_}_grib1.cml",)
+        )
+
+
 if __name__ == "__main__":
     tests.main()

src/iris_grib/tests/results/integration/load_convert/sample_file_loads/bulletin_40bytes.cml

@@ -0,0 +1,40 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" long_name="icao_standard_atmosphere_reference_height" units="m">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB2:d000c003n003"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord id="1d45e087" points="[15]" shape="(1,)" standard_name="forecast_period" units="Unit('hours')" value_type="int64"/>
+      </coord>
+      <coord>
+        <dimCoord id="9c8bdf81" points="[367902.]" shape="(1,)" standard_name="forecast_reference_time" units="Unit('hours since 1970-01-01 00:00:00', calendar='standard')" value_type="float64"/>
+      </coord>
+      <coord datadims="[0]">
+        <dimCoord id="77a50eb5" points="[ 90.  ,  88.75,  87.5 , ..., -87.5 , -88.75,
+ -90.  ]" shape="(145,)" standard_name="latitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6371229.0"/>
+        </dimCoord>
+      </coord>
+      <coord datadims="[1]">
+        <dimCoord circular="True" id="f913a8b3" points="[  0.  ,   1.25,   2.5 , ..., 356.25, 357.5 ,
+ 358.75]" shape="(288,)" standard_name="longitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6371229.0"/>
+        </dimCoord>
+      </coord>
+      <coord>
+        <dimCoord id="cb784457" points="[367917.]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='standard')" value_type="float64"/>
+      </coord>
+      <coord>
+        <dimCoord id="e2d6f010" points="[0.]" shape="(1,)" units="Unit('unknown')" value_type="float64">
+          <attributes>
+            <attribute name="GRIB_fixed_surface_type" value="11"/>
+          </attributes>
+        </dimCoord>
+      </coord>
+    </coords>
+    <cellMethods/>
+    <data checksum="0x47eb69cc" dtype="float64" mask_checksum="0x12cff8b4" shape="(145, 288)"/>
+  </cube>
+</cubes>

src/iris_grib/tests/results/integration/load_convert/sample_file_loads/bulletin_41bytes.cml

@@ -0,0 +1,47 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" units="unknown">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB1:t001c085n002"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord id="1d45e087" points="[72]" shape="(1,)" standard_name="forecast_period" units="Unit('hours')" value_type="int32"/>
+      </coord>
+      <coord datadims="[0]">
+        <dimCoord id="98bcd589" points="[ 45. ,  43.5,  42. ,  40.5,  39. ,  37.5,  36. ,
+  34.5,  33. ,  31.5,  30. ,  28.5,  27. ,  25.5,
+  24. ,  22.5,  21. ,  19.5,  18. ,  16.5,  15. ,
+  13.5,  12. ,  10.5,   9. ,   7.5,   6. ,   4.5,
+   3. ,   1.5,   0. ,  -1.5,  -3. ,  -4.5,  -6. ,
+  -7.5,  -9. , -10.5, -12. , -13.5, -15. , -16.5,
+ -18. , -19.5, -21. , -22.5, -24. , -25.5, -27. ,
+ -28.5, -30. , -31.5, -33. , -34.5, -36. , -37.5,
+ -39. , -40.5, -42. , -43.5, -45. ]" shape="(61,)" standard_name="latitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6367470.0"/>
+        </dimCoord>
+      </coord>
+      <coord datadims="[1]">
+        <dimCoord id="160a738f" points="[-30. , -28.5, -27. , -25.5, -24. , -22.5, -21. ,
+ -19.5, -18. , -16.5, -15. , -13.5, -12. , -10.5,
+  -9. ,  -7.5,  -6. ,  -4.5,  -3. ,  -1.5,   0. ,
+   1.5,   3. ,   4.5,   6. ,   7.5,   9. ,  10.5,
+  12. ,  13.5,  15. ,  16.5,  18. ,  19.5,  21. ,
+  22.5,  24. ,  25.5,  27. ,  28.5,  30. ,  31.5,
+  33. ,  34.5,  36. ,  37.5,  39. ,  40.5,  42. ,
+  43.5,  45. ,  46.5,  48. ,  49.5,  51. ,  52.5,
+  54. ,  55.5,  57. ,  58.5,  60. ]" shape="(61,)" standard_name="longitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6367470.0"/>
+        </dimCoord>
+      </coord>
+      <coord>
+        <auxCoord id="61bde96d" long_name="originating_centre" points="['unknown centre lfpw']" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
+      </coord>
+      <coord>
+        <dimCoord id="cb784457" points="[379872.]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='standard')" value_type="float64"/>
+      </coord>
+    </coords>
+    <cellMethods/>
+    <data checksum="0x6b50c19a" dtype="float64" shape="(61, 61)"/>
+  </cube>
+</cubes>

src/iris_grib/tests/results/integration/load_convert/sample_file_loads/mapped_cf_data_grib1.cml

@@ -0,0 +1,34 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" long_name="sea_surface_temperature" standard_name="sea_surface_temperature" units="K">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB1:t128c098n034"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord id="1d45e087" points="[0]" shape="(1,)" standard_name="forecast_period" units="Unit('hours')" value_type="int32"/>
+      </coord>
+      <coord datadims="[0]">
+        <dimCoord id="98bcd589" points="[ 90.,  89.,  88., ..., -88., -89., -90.]" shape="(181,)" standard_name="latitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6367470.0"/>
+        </dimCoord>
+      </coord>
+      <coord datadims="[1]">
+        <dimCoord circular="True" id="160a738f" points="[  0.,   1.,   2., ..., 357., 358., 359.]" shape="(360,)" standard_name="longitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6367470.0"/>
+        </dimCoord>
+      </coord>
+      <coord>
+        <auxCoord id="61bde96d" long_name="originating_centre" points="['European Centre for Medium Range Weather Forecasts']" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
+      </coord>
+      <coord>
+        <dimCoord id="6eef7051" long_name="pressure" points="[500]" shape="(1,)" units="Unit('hPa')" value_type="int32"/>
+      </coord>
+      <coord>
+        <dimCoord id="cb784457" points="[317364.]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='standard')" value_type="float64"/>
+      </coord>
+    </coords>
+    <cellMethods/>
+    <data checksum="0xaf017271" dtype="float64" shape="(181, 360)"/>
+  </cube>
+</cubes>

src/iris_grib/tests/results/integration/load_convert/sample_file_loads/polar_stereo_south_grib1.cml

@@ -0,0 +1,39 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" units="unknown">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB1:t002c007n061"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord bounds="[[0., 1.]]" id="1d45e087" points="[0.5]" shape="(1,)" standard_name="forecast_period" units="Unit('hours')" value_type="float64"/>
+      </coord>
+      <coord>
+        <auxCoord id="61bde96d" long_name="originating_centre" points="['US National Weather Service, National Centres for Environmental Prediction']" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
+      </coord>
+      <coord datadims="[1]">
+        <dimCoord id="71946cd5" points="[-1901416.77987822, -1896653.77987822,
+ -1891890.77987822, ...,  3423617.22012178,
+  3428380.22012178,  3433143.22012178]" shape="(1121,)" standard_name="projection_x_coordinate" units="Unit('m')" value_type="float64">
+          <polarStereographic central_lat="-90.0" central_lon="-105.0" ellipsoid="GeogCS(6367470.0)" false_easting="0.0" false_northing="0.0" scale_factor_at_projection_origin="None" true_scale_lat="60.0"/>
+        </dimCoord>
+      </coord>
+      <coord datadims="[0]">
+        <dimCoord id="88c2c643" points="[-7613145.49149694, -7608382.49149694,
+ -7603619.49149694, ..., -3431231.49149694,
+ -3426468.49149694, -3421705.49149694]" shape="(881,)" standard_name="projection_y_coordinate" units="Unit('m')" value_type="float64">
+          <polarStereographic central_lat="-90.0" central_lon="-105.0" ellipsoid="GeogCS(6367470.0)" false_easting="0.0" false_northing="0.0" scale_factor_at_projection_origin="None" true_scale_lat="60.0"/>
+        </dimCoord>
+      </coord>
+      <coord>
+        <dimCoord bounds="[[380337., 380338.]]" id="cb784457" points="[380337.5]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='standard')" value_type="float64"/>
+      </coord>
+    </coords>
+    <cellMethods>
+      <cellMethod method="sum">
+        <coord name="time"/>
+      </cellMethod>
+    </cellMethods>
+    <data checksum="0x75146405" dtype="float64" mask_checksum="0xdbe0da54" shape="(881, 1121)"/>
+  </cube>
+</cubes>

src/iris_grib/tests/results/integration/load_convert/sample_file_loads/reduced_gg_grib1.cml

@@ -0,0 +1,45 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" long_name="air_temperature" standard_name="air_temperature" units="K">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB1:t128c098n130"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord id="1d45e087" points="[0]" shape="(1,)" standard_name="forecast_period" units="Unit('hours')" value_type="int32"/>
+      </coord>
+      <coord datadims="[0]">
+        <auxCoord id="98bcd589" points="[ 87.86379884,  87.86379884,  87.86379884, ...,
+ -87.86379884, -87.86379884, -87.86379884]" shape="(6114,)" standard_name="latitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6367470.0"/>
+        </auxCoord>
+      </coord>
+      <coord>
+        <dimCoord id="c9091dc6" long_name="level_pressure" points="[2.00004005]" shape="(1,)" units="Unit('Pa')" value_type="float64"/>
+      </coord>
+      <coord datadims="[0]">
+        <auxCoord id="160a738f" points="[  0.,  18.,  36., ..., 306., 324., 342.]" shape="(6114,)" standard_name="longitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6367470.0"/>
+        </auxCoord>
+      </coord>
+      <coord>
+        <dimCoord id="38fd0305" points="[1]" shape="(1,)" standard_name="model_level_number" units="Unit('1')" value_type="int32">
+          <attributes>
+            <attribute name="positive" value="up"/>
+          </attributes>
+        </dimCoord>
+      </coord>
+      <coord>
+        <auxCoord id="61bde96d" long_name="originating_centre" points="['European Centre for Medium Range Weather Forecasts']" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
+      </coord>
+      <coord>
+        <dimCoord id="a5c170db" long_name="sigma" points="[0.]" shape="(1,)" units="Unit('1')" value_type="float64"/>
+      </coord>
+      <coord>
+        <dimCoord id="cb784457" points="[326292.]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='standard')" value_type="float64"/>
+      </coord>
+    </coords>
+    <cellMethods/>
+    <data checksum="0x6fe0ee2c" dtype="float64" shape="(6114,)"/>
+  </cube>
+</cubes>

src/iris_grib/tests/results/integration/load_convert/sample_file_loads/second_order_packing.cml

@@ -0,0 +1,31 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" units="unknown">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB1:t001c085n008"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord id="1d45e087" points="[0]" shape="(1,)" standard_name="forecast_period" units="Unit('hours')" value_type="int32"/>
+      </coord>
+      <coord datadims="[0]">
+        <dimCoord id="98bcd589" points="[57. , 56.9, 56.8, ..., 35.2, 35.1, 35. ]" shape="(221,)" standard_name="latitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6367470.0"/>
+        </dimCoord>
+      </coord>
+      <coord datadims="[1]">
+        <dimCoord id="160a738f" points="[-11. , -10.9, -10.8, ...,  16.8,  16.9,  17. ]" shape="(281,)" standard_name="longitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6367470.0"/>
+        </dimCoord>
+      </coord>
+      <coord>
+        <auxCoord id="61bde96d" long_name="originating_centre" points="['unknown centre lfpw']" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
+      </coord>
+      <coord>
+        <dimCoord id="cb784457" points="[-17259552.]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='standard')" value_type="float64"/>
+      </coord>
+    </coords>
+    <cellMethods/>
+    <data checksum="0x7c584569" dtype="float64" shape="(221, 281)"/>
+  </cube>
+</cubes>