Skip to content

Commit c244d05

Browse files
douglatornelldouglatornell
authored
Refactor grib_to_netcdf time_counter calculation (#338)
* Refactor grib_to_netcdf time_counter calculation Update `time_counter` calculation to use forecast step ranges. This mitigates the fact that `cfgrib` no longer provides correct forecast time step values from ECCC GRIB2 files. This is a rework of commit 406e7e5 that tried to resolve the issue using the `valid_time` GRIB variable. This change makes our timestamp processing compatible with non-hourly step handling that was introduced in `cfgrib=0.9.12.0`. This change removes the need for version pinning of `cfgrib=0.9.11.0` that was introduced in PR#276. By doing that, issue #336 is resolved. * Update fixture to include fcst_step_range parameter Added the `fcst_step_range` parameter to the `mock_calc_nemo_var_ds()` test fixture. This change ensures alignment with the updated function signature and fixes test failures. * Use "python" instead of "python3" in history attr Updated the history attribute in `grib_to_netcdf.py` and its associated test to use "python" instead of "python3".
1 parent 9e4edb8 commit c244d05

8 files changed

+68
-43
lines changed

‎envs/environment-dev.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ dependencies:
2828
- beautifulsoup4
2929
- bottleneck
3030
- cartopy
31-
- cfgrib=0.9.11.0
31+
- cfgrib
3232
- click
3333
- cliff
3434
- cmocean
@@ -76,7 +76,7 @@ dependencies:
7676
- utm
7777
- verboselogs
7878
- watchdog
79-
- xarray=2024.7.0
79+
- xarray=2025.1.1
8080

8181
# For coding style, repo QA, and pkg management
8282
- black

‎envs/environment-fig-dev.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ dependencies:
2525
- beautifulsoup4
2626
- bottleneck
2727
- cartopy
28-
- cfgrib=0.9.11.0
28+
- cfgrib
2929
- cliff
3030
- cmocean
3131
- coloredlogs
@@ -60,7 +60,7 @@ dependencies:
6060
- tqdm
6161
- verboselogs
6262
- watchdog
63-
- xarray=2024.7.0
63+
- xarray=2025.1.1
6464

6565
# For code style & repo QA
6666
- pre-commit

‎envs/environment-linkcheck.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ dependencies:
1616
- beautifulsoup4
1717
- bottleneck
1818
- cartopy
19-
- cfgrib=0.9.11.0
19+
- cfgrib
2020
- click
2121
- cliff
2222
- cmocean
@@ -63,7 +63,7 @@ dependencies:
6363
- utm
6464
- verboselogs
6565
- watchdog
66-
- xarray=2024.7.0
66+
- xarray=2025.1.1
6767

6868
# For documentation links checking
6969
- sphinx==8.1.3

‎envs/environment-prod.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ dependencies:
2828
- beautifulsoup4
2929
- bottleneck
3030
- cartopy
31-
- cfgrib=0.9.11.0
31+
- cfgrib
3232
- click
3333
- cliff
3434
- cmocean
@@ -76,7 +76,7 @@ dependencies:
7676
- utm
7777
- verboselogs
7878
- watchdog
79-
- xarray=2024.7.0
79+
- xarray=2025.1.1
8080

8181
- pip:
8282
- angles

‎envs/environment-test.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ dependencies:
1616
- beautifulsoup4
1717
- bottleneck
1818
- cartopy
19-
- cfgrib=0.9.11.0
19+
- cfgrib
2020
- click
2121
- cliff
2222
- cmocean
@@ -62,7 +62,7 @@ dependencies:
6262
- utm
6363
- verboselogs
6464
- watchdog
65-
- xarray=2024.7.0
65+
- xarray=2025.1.1
6666

6767
# For unit tests and coverage monitoring
6868
- pytest

‎envs/requirements.txt

+23-23
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,13 @@ argon2-cffi-bindings==21.2.0
1515
arrow==1.3.0
1616
asttokens==3.0.0
1717
async-lru==2.0.4
18-
attrs==24.3.0
18+
attrs==25.1.0
1919
autopage==0.5.2
2020
babel==2.16.0
2121
backports.tarfile==1.2.0
2222
bcrypt==4.2.1
23-
beautifulsoup4==4.12.3
24-
black==24.10.0
23+
beautifulsoup4==4.13.3
24+
black==25.1.0
2525
bleach==6.2.0
2626
bokeh==3.6.2
2727
Bottleneck==1.4.2
@@ -30,7 +30,7 @@ cached-property==1.5.2
3030
Cartopy==0.24.0
3131
certifi==2025.1.31
3232
cffi==1.17.1
33-
cfgrib==0.9.11.0
33+
cfgrib==0.9.15.0
3434
cfgv==3.3.1
3535
cftime==1.6.4
3636
charset-normalizer==3.4.1
@@ -48,13 +48,13 @@ coverage==7.6.10
4848
cryptography==44.0.1
4949
cycler==0.12.1
5050
cytoolz==1.0.1
51-
dask==2024.12.1
52-
dask-expr==1.1.21
51+
dask==2025.2.0
52+
dask-expr==2.0.0
5353
debugpy==1.8.11
5454
decorator==5.1.1
5555
defusedxml==0.7.1
5656
distlib==0.3.9
57-
distributed==2024.12.1
57+
distributed==2025.2.0
5858
docutils==0.21.2
5959
eccodes==2.37.0
6060
editables==0.5
@@ -68,7 +68,7 @@ fastjsonschema==2.21.1
6868
feedgen==1.0.0
6969
filelock==3.16.1
7070
findlibs==0.0.5
71-
flox==0.9.15
71+
flox==0.10.0
7272
fonttools==4.55.3
7373
fqdn==1.5.1
7474
fsspec==2024.12.0
@@ -79,7 +79,7 @@ GitPython==3.1.44
7979
gsw==3.6.19
8080
h11==0.14.0
8181
h2==4.1.0
82-
h5netcdf==1.4.1
82+
h5netcdf==1.5.0
8383
h5py==3.12.1
8484
hatch==1.14.0
8585
hatchling==1.27.0
@@ -121,9 +121,9 @@ jupyterlab_server==2.27.3
121121
keyring==25.6.0
122122
kiwisolver==1.4.7
123123
locket==1.0.0
124-
lxml==5.3.0
124+
lxml==5.3.1
125125
lz4==4.3.3
126-
Mako==1.3.8
126+
Mako==1.3.9
127127
markdown-it-py==3.0.0
128128
MarkupSafe==3.0.2
129129
matplotlib==3.10.0
@@ -146,7 +146,7 @@ netCDF4==1.7.2
146146
nodeenv==1.9.1
147147
notebook==7.3.2
148148
notebook_shim==0.2.4
149-
numpy==2.2.1
149+
numpy==2.2.3
150150
numpy-groupies==0.11.2
151151
numpy-indexed==0.3.7
152152
openpyxl==3.1.5
@@ -155,19 +155,19 @@ overrides==7.7.0
155155
packaging==24.2
156156
pandas==2.2.3
157157
pandocfilters==1.5.0
158-
paramiko==3.5.0
158+
paramiko==3.5.1
159159
parso==0.8.4
160160
partd==1.4.2
161161
pathspec==0.12.1
162162
pbr==6.1.0
163163
pexpect==4.9.0
164164
pickleshare==0.7.5
165165
pillow==11.1.0
166-
pip==24.3.1
166+
pip==25.0.1
167167
pkgutil_resolve_name==1.3.10
168168
platformdirs==4.3.6
169169
pluggy==1.5.0
170-
pre_commit==4.0.1
170+
pre_commit==4.1.0
171171
prettytable==3.12.0
172172
prometheus_client==0.21.1
173173
prompt_toolkit==3.0.48
@@ -180,9 +180,9 @@ Pygments==2.19.1
180180
pygrib==2.1.6
181181
PyNaCl==1.5.0
182182
pyparsing==3.2.1
183-
pypdf==5.1.0
183+
pypdf==5.3.0
184184
pyperclip==1.9.0
185-
pyproj==3.7.0
185+
pyproj==3.7.1
186186
pyshp==2.3.1
187187
PySide6==6.8.1
188188
PySocks==1.7.1
@@ -196,13 +196,13 @@ python-hglib==2.6.2
196196
python-json-logger==2.0.7
197197
pytz==2024.1
198198
PyYAML==6.0.2
199-
pyzmq==26.2.0
199+
pyzmq==26.2.1
200200
referencing==0.35.1
201201
requests==2.32.3
202202
requests-file==2.1.0
203203
requests-toolbelt==1.0.0
204204
Reshapr==25.1.dev0
205-
retrying==1.3.3
205+
retrying==1.3.4
206206
rfc3339_validator==0.1.4
207207
rfc3986-validator==0.1.1
208208
rich==13.9.4
@@ -211,11 +211,11 @@ SalishSeaCmd==25.1.dev0
211211
SalishSeaNowcast==25.1.dev0
212212
SalishSeaTools==25.1.dev0
213213
schedule==1.2.2
214-
scipy==1.15.1
214+
scipy==1.15.2
215215
scour==0.38.2
216216
SecretStorage==3.3.3
217217
Send2Trash==1.8.3
218-
sentry-sdk==2.20.0
218+
sentry-sdk==2.22.0
219219
setuptools==75.8.0
220220
shapely==2.0.6
221221
shellingham==1.5.4
@@ -238,7 +238,7 @@ sphinxcontrib-qthelp==2.0.0
238238
sphinxcontrib-serializinghtml==1.1.10
239239
stack_data==0.6.3
240240
stevedore==5.4.0
241-
structlog==24.4.0
241+
structlog==25.1.0
242242
supervisor==4.2.5
243243
sysrsync==1.1.1
244244
tblib==3.0.0
@@ -270,7 +270,7 @@ wcwidth==0.2.13
270270
webcolors==24.11.1
271271
webencodings==0.5.1
272272
websocket-client==1.8.0
273-
xarray==2024.7.0
273+
xarray==2025.1.1
274274
xyzservices==2024.9.0
275275
zeep==4.3.1
276276
zict==3.0.0

‎nowcast/workers/grib_to_netcdf.py

+25-7
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,14 @@ def _calc_nemo_ds(
266266
fcst_date, fcst_hr, fcst_step_range, msc_var, full_grid, config
267267
)
268268
nemo_datasets[nemo_var] = _calc_nemo_var_ds(
269-
msc_var, grib_var, nemo_var, grib_files, full_grid, georef_ds, config
269+
msc_var,
270+
grib_var,
271+
nemo_var,
272+
grib_files,
273+
fcst_step_range,
274+
full_grid,
275+
georef_ds,
276+
config,
270277
)
271278
nemo_ds = xarray.combine_by_coords(
272279
nemo_datasets.values(), combine_attrs="drop_conflicts"
@@ -328,7 +335,7 @@ def _trim_grib(ds, y_slice, x_slice):
328335
# Select region of interest
329336
ds = ds.sel(y=y_slice, x=x_slice)
330337
# Drop coordinates that we don't need
331-
keep_coords = ("time", "step", "latitude", "longitude")
338+
keep_coords = ("time", "step", "valid_time", "latitude", "longitude")
332339
ds = ds.reset_coords(
333340
[coord for coord in ds.coords if coord not in keep_coords],
334341
drop=True,
@@ -337,13 +344,21 @@ def _trim_grib(ds, y_slice, x_slice):
337344

338345

339346
def _calc_nemo_var_ds(
340-
msc_var, grib_var, nemo_var, grib_files, full_grid, georef_ds, config
347+
msc_var,
348+
grib_var,
349+
nemo_var,
350+
grib_files,
351+
fcst_step_range,
352+
full_grid,
353+
georef_ds,
354+
config,
341355
):
342356
"""
343357
:param str msc_var:
344358
:param str grib_var:
345359
:param str nemo_var:
346360
:param list grib_files:
361+
:param tuple fcst_step_range:
347362
:param boolean full_grid:
348363
:param :py:class:`xarray.Dataset` or None georef_ds:
349364
:param dict config:
@@ -373,7 +388,10 @@ def _calc_nemo_var_ds(
373388
engine="cfgrib",
374389
backend_kwargs={"indexpath": ""},
375390
)
376-
time_counter = grib_ds.step.values + grib_ds.time.values
391+
start, stop = fcst_step_range
392+
time_counter = grib_ds.time.values + numpy.array(
393+
[numpy.timedelta64(fcst_step, "h") for fcst_step in range(start, stop + 1)]
394+
)
377395
nemo_da = xarray.DataArray(
378396
data=grib_ds[grib_var].data,
379397
coords={
@@ -413,8 +431,8 @@ def _calc_nemo_var_ds(
413431
# Drop unneeded variables that come from full continental domain GRIB files.
414432
# Drop time separately from lons/lats because drop_vars() fails is any of the vars
415433
# in the list don't exist.
416-
nemo_ds = nemo_ds.drop_vars(["time"])
417-
nemo_ds = nemo_ds.drop_vars(["longitude", "latitude"])
434+
nemo_ds = nemo_ds.drop_vars(["time", "valid_time"])
435+
nemo_ds = nemo_ds.drop_vars(["latitude", "longitude"])
418436
except ValueError:
419437
# We don't care if some or all of them don't exist in the dataset
420438
pass
@@ -728,7 +746,7 @@ def _write_netcdf(nemo_ds, file_date, run_date, run_type, config, fcst=False):
728746
{
729747
"history": (
730748
f"[{arrow.now('local').format('ddd YYYY-MM-DD HH:mm:ss ZZ')}] "
731-
f"python3 -m nowcast.workers.grib_to_netcdf $NOWCAST_YAML "
749+
f"python -m nowcast.workers.grib_to_netcdf $NOWCAST_YAML "
732750
f"{run_type} --run-date {run_date.format('YYYY-MM-DD')}"
733751
),
734752
}

‎tests/workers/test_grib_to_netcdf.py

+10-3
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ def config(base_config):
5252
- [VGRD_AGL-10m, v10, v_wind] # v component of wind velocity at 10m elevation
5353
- [DSWRF_Sfc, ssrd, solar] # accumulated downward shortwave (solar) radiation at ground level
5454
- [DLWRF_Sfc, strd, therm_rad] # accumulated downward longwave (thermal) radiation at ground level
55-
- [LHTFL_Sfc, lhtfl, LHTFL_surface] # upward surface latent heat flux (for VHFR FVCOM)
55+
- [LHTFL_Sfc, slhtf, LHTFL_surface] # upward surface latent heat flux (for VHFR FVCOM)
5656
- [TMP_AGL-2m, t2m, tair] # air temperature at 2m elevation
5757
- [SPFH_AGL-2m, sh2, qair] # specific humidity at 2m elevation
5858
- [RH_AGL-2m, r2, RH_2maboveground] # relative humidity at 2m elevation (for VHFR FVCOM)
@@ -248,7 +248,14 @@ def _mock_open_dataset(path):
248248
@pytest.fixture
249249
def mock_calc_nemo_var_ds(monkeypatch):
250250
def _mock_calc_nemo_var_ds(
251-
msc_var, grib_var, nemo_var, grib_files, full_grid, georef_ds, config
251+
msc_var,
252+
grib_var,
253+
nemo_var,
254+
grib_files,
255+
fcst_step_range,
256+
full_grid,
257+
georef_ds,
258+
config,
252259
):
253260
pass
254261

@@ -842,7 +849,7 @@ def mock_now(tz):
842849

843850
expected = (
844851
f"[Sun 2023-03-12 16:02:43 -07:00] "
845-
f"python3 -m nowcast.workers.grib_to_netcdf $NOWCAST_YAML "
852+
f"python -m nowcast.workers.grib_to_netcdf $NOWCAST_YAML "
846853
f"{run_type} --run-date 2023-03-12"
847854
)
848855
assert nemo_ds.attrs["history"] == expected

0 commit comments

Comments
 (0)