Fix issues that cause crop_gribs worker to fail after updating production env to Python 3.13 (#337)

douglatornell · web-flow · commit 587d91cb2759 · 2025-02-23T09:31:05.000-08:00
* Pin xarray version to 2024.7.0 in all environments Downgraded xarray dependency across all environment files to version 2024.7.0 to avoid TypeError in `crop_gribs` due to addition of `engine` arg to `xarray.backends.api._validate_attrs()` in 2024.9.0 (see pydata/xarray#9407). That change is incompatible with cfgrib=0.9.11.0 that we have to pin to because later versions result in atmospheric forcing netCDF files in which all the `time_counter` values are the same (see work journal notes from 15-Jun-2024). * Update LHTFL_Sfc variable alias to 'slhtf' Replaced the alias 'lhtfl' with 'slhtf' for the LHTFL_Sfc variable across multiple test files and the configuration file. It appears that 'slhtf' became the only acceptable GRIB2 standard name for the upward surface latent heat flux variable sometime after 15-Jun-2024.
diff --git a/config/nowcast.yaml b/config/nowcast.yaml
@@ -37,7 +37,7 @@ weather:
         - [VGRD_AGL-10m, v10, v_wind]           # v component of wind velocity at 10m elevation
         - [DSWRF_Sfc, ssrd, solar]              # accumulated downward shortwave (solar) radiation at ground level
         - [DLWRF_Sfc, strd, therm_rad]          # accumulated downward longwave (thermal) radiation at ground level
-        - [LHTFL_Sfc, lhtfl, LHTFL_surface]     # upward surface latent heat flux (for VHFR FVCOM)
+        - [LHTFL_Sfc, slhtf, LHTFL_surface]     # upward surface latent heat flux (for VHFR FVCOM)
         - [TMP_AGL-2m, t2m, tair]               # air temperature at 2m elevation
         - [SPFH_AGL-2m, sh2, qair]              # specific humidity at 2m elevation
         - [RH_AGL-2m, r2, RH_2maboveground]     # relative humidity at 2m elevation (for VHFR FVCOM)
diff --git a/envs/environment-dev.yaml b/envs/environment-dev.yaml
@@ -76,7 +76,7 @@ dependencies:
   - utm
   - verboselogs
   - watchdog
-  - xarray
+  - xarray=2024.7.0
 
   # For coding style, repo QA, and pkg management
   - black
diff --git a/envs/environment-fig-dev.yaml b/envs/environment-fig-dev.yaml
@@ -60,7 +60,7 @@ dependencies:
   - tqdm
   - verboselogs
   - watchdog
-  - xarray
+  - xarray=2024.7.0
 
   # For code style & repo QA
   - pre-commit
diff --git a/envs/environment-linkcheck.yaml b/envs/environment-linkcheck.yaml
@@ -63,7 +63,7 @@ dependencies:
   - utm
   - verboselogs
   - watchdog
-  - xarray
+  - xarray=2024.7.0
 
   # For documentation links checking
   - sphinx==8.1.3
diff --git a/envs/environment-prod.yaml b/envs/environment-prod.yaml
@@ -76,7 +76,7 @@ dependencies:
   - utm
   - verboselogs
   - watchdog
-  - xarray
+  - xarray=2024.7.0
 
   - pip:
     - angles
diff --git a/envs/environment-test.yaml b/envs/environment-test.yaml
@@ -62,7 +62,7 @@ dependencies:
   - utm
   - verboselogs
   - watchdog
-  - xarray
+  - xarray=2024.7.0
 
   # For unit tests and coverage monitoring
   - pytest
diff --git a/envs/requirements.txt b/envs/requirements.txt
@@ -10,20 +10,25 @@
 alabaster==1.0.0
 angles==2.0
 anyio==4.8.0
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
 arrow==1.3.0
+asttokens==3.0.0
+async-lru==2.0.4
 attrs==24.3.0
 autopage==0.5.2
 babel==2.16.0
 backports.tarfile==1.2.0
 bcrypt==4.2.1
 beautifulsoup4==4.12.3
 black==24.10.0
+bleach==6.2.0
 bokeh==3.6.2
 Bottleneck==1.4.2
 Brotli==1.1.0
 cached-property==1.5.2
 Cartopy==0.24.0
-certifi==2024.12.14
+certifi==2025.1.31
 cffi==1.17.1
 cfgrib==0.9.11.0
 cfgv==3.3.1
@@ -37,27 +42,35 @@ cmocean==4.0.3
 colorama==0.4.6
 coloredlogs==15.0.1
 colorspacious==1.1.2
+comm==0.2.2
 contourpy==1.3.1
 coverage==7.6.10
 cryptography==44.0.1
 cycler==0.12.1
 cytoolz==1.0.1
 dask==2024.12.1
 dask-expr==1.1.21
+debugpy==1.8.11
+decorator==5.1.1
+defusedxml==0.7.1
 distlib==0.3.9
 distributed==2024.12.1
 docutils==0.21.2
 eccodes==2.37.0
 editables==0.5
+entrypoints==0.4
 et_xmlfile==2.0.0
 exceptiongroup==1.2.2
 execnet==2.1.1
+executing==2.1.0
 f90nml==1.4.4
+fastjsonschema==2.21.1
 feedgen==1.0.0
 filelock==3.16.1
 findlibs==0.0.5
 flox==0.9.15
 fonttools==4.55.3
+fqdn==1.5.1
 fsspec==2024.12.0
 future==1.0.0
 FVCOM-Cmd==0.1
@@ -82,12 +95,29 @@ imagesize==1.4.1
 importlib_metadata==8.5.0
 importlib_resources==6.5.2
 iniconfig==2.0.0
+ipykernel==6.29.5
+ipython==8.31.0
 isodate==0.7.2
+isoduration==20.11.0
 jaraco.classes==3.4.0
 jaraco.context==6.0.1
 jaraco.functools==4.1.0
+jedi==0.19.2
 jeepney==0.8.0
 Jinja2==3.1.5
+json5==0.10.0
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter-events==0.11.0
+jupyter-lsp==2.2.5
+jupyter_server==2.15.0
+jupyter_server_terminals==0.5.3
+jupyterlab==4.3.4
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
 keyring==25.6.0
 kiwisolver==1.4.7
 locket==1.0.0
@@ -97,37 +127,53 @@ Mako==1.3.8
 markdown-it-py==3.0.0
 MarkupSafe==3.0.2
 matplotlib==3.10.0
+matplotlib-inline==0.1.7
 mdurl==0.1.2
+mistune==3.1.0
 moad_tools==25.2.dev0
 more-itertools==10.6.0
 msgpack==1.1.0
 munkres==1.1.4
 mypy_extensions==1.0.0
+nbclient==0.10.2
+nbconvert==7.16.5
+nbformat==5.10.4
 nc-time-axis==1.4.1
 NEMO-Cmd==25.1.dev0
 NEMO_Nowcast==25.1.dev0
+nest_asyncio==1.6.0
 netCDF4==1.7.2
 nodeenv==1.9.1
+notebook==7.3.2
+notebook_shim==0.2.4
 numpy==2.2.1
 numpy-groupies==0.11.2
 numpy-indexed==0.3.7
 openpyxl==3.1.5
 OPPTools==0.1.0
+overrides==7.7.0
 packaging==24.2
 pandas==2.2.3
+pandocfilters==1.5.0
 paramiko==3.5.0
+parso==0.8.4
 partd==1.4.2
 pathspec==0.12.1
 pbr==6.1.0
 pexpect==4.9.0
+pickleshare==0.7.5
 pillow==11.1.0
 pip==24.3.1
+pkgutil_resolve_name==1.3.10
 platformdirs==4.3.6
 pluggy==1.5.0
 pre_commit==4.0.1
 prettytable==3.12.0
+prometheus_client==0.21.1
+prompt_toolkit==3.0.48
 psutil==6.1.1
 ptyprocess==0.7.0
+pure_eval==0.2.3
 pyarrow==18.1.0
 pycparser==2.22
 Pygments==2.19.1
@@ -147,22 +193,28 @@ pytest-randomly==3.15.0
 pytest-xdist==3.6.1
 python-dateutil==2.9.0.post0
 python-hglib==2.6.2
+python-json-logger==2.0.7
 pytz==2024.1
 PyYAML==6.0.2
 pyzmq==26.2.0
+referencing==0.35.1
 requests==2.32.3
 requests-file==2.1.0
 requests-toolbelt==1.0.0
 Reshapr==25.1.dev0
 retrying==1.3.3
+rfc3339_validator==0.1.4
+rfc3986-validator==0.1.1
 rich==13.9.4
+rpds-py==0.22.3
 SalishSeaCmd==25.1.dev0
 SalishSeaNowcast==25.1.dev0
 SalishSeaTools==25.1.dev0
 schedule==1.2.2
 scipy==1.15.1
 scour==0.38.2
 SecretStorage==3.3.3
+Send2Trash==1.8.3
 sentry-sdk==2.20.0
 setuptools==75.8.0
 shapely==2.0.6
@@ -184,32 +236,41 @@ sphinxcontrib-jquery==4.1
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==2.0.0
 sphinxcontrib-serializinghtml==1.1.10
+stack_data==0.6.3
 stevedore==5.4.0
 structlog==24.4.0
 supervisor==4.2.5
 sysrsync==1.1.1
 tblib==3.0.0
 tenacity==9.0.0
+terminado==0.18.1
+tinycss2==1.4.0
 toml==0.10.2
 tomli==2.2.1
 tomli_w==1.2.0
 tomlkit==0.13.2
 toolz==1.0.0
 tornado==6.4.2
 tqdm==4.67.1
+traitlets==5.14.3
 trove-classifiers==2025.1.10.15
 types-python-dateutil==2.9.0.20241206
 typing_extensions==4.12.2
+typing_utils==0.1.0
 tzdata==2024.2
 ukkonen==1.0.1
+uri-template==1.3.0
 urllib3==2.3.0
 userpath==1.9.2
 utm==0.7.0
 verboselogs==1.7
 virtualenv==20.28.1
 watchdog==6.0.0
 wcwidth==0.2.13
-xarray==2025.1.1
+webcolors==24.11.1
+webencodings==0.5.1
+websocket-client==1.8.0
+xarray==2024.7.0
 xyzservices==2024.9.0
 zeep==4.3.1
 zict==3.0.0
diff --git a/tests/workers/test_collect_weather.py b/tests/workers/test_collect_weather.py
@@ -182,7 +182,7 @@ def test_weather_download_2_5_km_section(self, prod_config):
             ["VGRD_AGL-10m", "v10", "v_wind"],
             ["DSWRF_Sfc", "ssrd", "solar"],
             ["DLWRF_Sfc", "strd", "therm_rad"],
-            ["LHTFL_Sfc", "lhtfl", "LHTFL_surface"],
+            ["LHTFL_Sfc", "slhtf", "LHTFL_surface"],
             ["TMP_AGL-2m", "t2m", "tair"],
             ["SPFH_AGL-2m", "sh2", "qair"],
             ["RH_AGL-2m", "r2", "RH_2maboveground"],
diff --git a/tests/workers/test_crop_gribs.py b/tests/workers/test_crop_gribs.py
@@ -172,7 +172,7 @@ def test_weather_download_2_5_km_section(self, prod_config):
             ["VGRD_AGL-10m", "v10", "v_wind"],
             ["DSWRF_Sfc", "ssrd", "solar"],
             ["DLWRF_Sfc", "strd", "therm_rad"],
-            ["LHTFL_Sfc", "lhtfl", "LHTFL_surface"],
+            ["LHTFL_Sfc", "slhtf", "LHTFL_surface"],
             ["TMP_AGL-2m", "t2m", "tair"],
             ["SPFH_AGL-2m", "sh2", "qair"],
             ["RH_AGL-2m", "r2", "RH_2maboveground"],
diff --git a/tests/workers/test_download_weather.py b/tests/workers/test_download_weather.py
@@ -180,7 +180,7 @@ def test_weather_download_2_5_km_section(self, prod_config):
             ["VGRD_AGL-10m", "v10", "v_wind"],
             ["DSWRF_Sfc", "ssrd", "solar"],
             ["DLWRF_Sfc", "strd", "therm_rad"],
-            ["LHTFL_Sfc", "lhtfl", "LHTFL_surface"],
+            ["LHTFL_Sfc", "slhtf", "LHTFL_surface"],
             ["TMP_AGL-2m", "t2m", "tair"],
             ["SPFH_AGL-2m", "sh2", "qair"],
             ["RH_AGL-2m", "r2", "RH_2maboveground"],
diff --git a/tests/workers/test_grib_to_netcdf.py b/tests/workers/test_grib_to_netcdf.py
@@ -168,7 +168,7 @@ def test_weather_download_2_5_km_section(self, prod_config):
             ["VGRD_AGL-10m", "v10", "v_wind"],
             ["DSWRF_Sfc", "ssrd", "solar"],
             ["DLWRF_Sfc", "strd", "therm_rad"],
-            ["LHTFL_Sfc", "lhtfl", "LHTFL_surface"],
+            ["LHTFL_Sfc", "slhtf", "LHTFL_surface"],
             ["TMP_AGL-2m", "t2m", "tair"],
             ["SPFH_AGL-2m", "sh2", "qair"],
             ["RH_AGL-2m", "r2", "RH_2maboveground"],
