Skip to content

Commit 6a287b7

Browse files
guillaumevernieresAnna ShlyaevaRussTreadon-NOAAaerorahul
authored
Add capability to run diff resolutions for marine anl and background (#3238)
This PR mostly includes configuration changes to allow the use a B-matrix at a defined resolution that may or may not be the same as the background. - Addition of an interpolation step in the B-matrix job - version update for the fix files - **the `gdas.cd` update should be the correct one now.** Resolves #3230 Resolves NOAA-EMC/GDASApp/issues/1375 # Dependencies - NOAA-EMC/GDASApp/pull/1441 - NOAA-EMC/jcb-gdas/pull/66 --------- Co-authored-by: Anna Shlyaeva <anna.v.shlyaeva@noaa.gov> Co-authored-by: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Co-authored-by: Rahul Mahajan <aerorahul@users.noreply.github.com>
1 parent a1c0e7d commit 6a287b7

14 files changed

+89
-42
lines changed

‎ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml

+3-2
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@ arguments:
88
resdetatmos: 384
99
resdetocean: 0.25
1010
nens: 0
11-
interval: 6
11+
interval: 0
1212
start: warm
1313
comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
1414
expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
1515
idate: 2021063018
1616
edate: 2021070306
1717
#icsdir: /scratch1/NCEPDEV/climate/Jessica.Meixner/cycling/IC_2021063000_V2
18-
icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
18+
#icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
19+
icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/hybrid-test/
1920
yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marine3dvar.yaml

‎ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,6 @@ arguments:
1515
expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
1616
idate: 2021063018
1717
edate: 2021070306
18-
icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
18+
#icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
19+
icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/hybrid-test/
1920
yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marinehyb.yaml

‎ci/cases/gfsv17/marine3dvar.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,12 @@ base:
1616
ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
1717
DO_TEST_MODE: "YES"
1818

19+
prepoceanobs:
20+
use_exp_obs: "YES"
21+
dmpdir_exp: /scratch1/NCEPDEV/da/common/
22+
1923
marineanl:
2024
SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca
25+
SOCA_ANL_GEOM: {{ HOMEgfs }}/fix/gdas/soca/720x540x75/soca
2126
SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
2227
SOCA_NINNER: 100

‎ci/cases/gfsv17/marinehyb.yaml

+6-1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ base:
1515
FHMAX_GFS: 240
1616
ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
1717

18+
prepoceanobs:
19+
use_exp_obs: "YES"
20+
dmpdir_exp: /scratch1/NCEPDEV/da/common/
21+
1822
marineanl:
1923
SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca
20-
SOCA_NINNER: 20 # revert to ~100 after the memory leak is fixed
24+
SOCA_ANL_GEOM: {{ HOMEgfs }}/fix/gdas/soca/720x540x75/soca
25+
SOCA_NINNER: 100

‎jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN

+5
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,11 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
2020
COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \
2121
COMOUT_ICE_ANALYSIS:COM_ICE_ANALYSIS_TMPL
2222

23+
export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}"
24+
export DATAens="${DATAjob}/ensdata"
25+
export GDUMP="gdas"
26+
export GDUMP_ENS="enkf${GDUMP}"
27+
2328
##############################################
2429
# Begin JOB SPECIFIC work
2530
##############################################

‎parm/config/gfs/config.marineanl

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@
1111
export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config"
1212
export MARINE_OBS_LIST_YAML=@SOCA_OBS_LIST@
1313
export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@
14+
export SOCA_ANL_GEOM=@SOCA_ANL_GEOM@
1415
export SOCA_NINNER=@SOCA_NINNER@
1516
export DOMAIN_STACK_SIZE=116640000 #TODO: Make the stack size resolution dependent
1617
export SOCA_ENS_BKG_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ens_bkg_stage.yaml.j2"

‎parm/config/gfs/config.resources

+19-16
Original file line numberDiff line numberDiff line change
@@ -509,23 +509,26 @@ case ${step} in
509509
walltime="00:10:00"
510510
ntasks=1
511511
threads_per_task=1
512-
tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
512+
tasks_per_node=${max_tasks_per_node}
513513
memory="24GB"
514514
;;
515515

516516
"prepoceanobs")
517517
walltime="00:10:00"
518518
ntasks=1
519519
threads_per_task=1
520-
tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
520+
tasks_per_node=${max_tasks_per_node}
521521
memory="48GB"
522522
;;
523523

524524
"marinebmat")
525525
npes=16
526526
ntasks=16
527527
case ${OCNRES} in
528-
"025") ntasks=480;;
528+
"025")
529+
ntasks=480
530+
memory="256GB"
531+
;;
529532
"050") ntasks=16;;
530533
"100") ntasks=16;;
531534
"500") ntasks=16;;
@@ -537,15 +540,15 @@ case ${step} in
537540
walltime="00:30:00"
538541
threads_per_task=1
539542
export is_exclusive=True
540-
tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
543+
tasks_per_node=$(( max_tasks_per_node / 2 ))
541544
;;
542545

543546
"marineanlvar")
544547
ntasks=16
545548
case ${OCNRES} in
546549
"025")
547550
ntasks=480
548-
memory="96GB"
551+
memory="256GB"
549552
;;
550553
"050")
551554
ntasks=16
@@ -564,18 +567,18 @@ case ${step} in
564567
exit 4
565568
esac
566569

567-
walltime="00:15:00"
570+
walltime="00:30:00"
568571
threads_per_task=1
569572
export is_exclusive=True
570-
tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
573+
tasks_per_node=$(( max_tasks_per_node / 2 ))
571574
;;
572575

573576
"ocnanalecen")
574577
ntasks=16
575578
case ${OCNRES} in
576579
"025")
577580
ntasks=40
578-
memory="96GB"
581+
memory="256GB"
579582
;;
580583
"050")
581584
ntasks=16
@@ -595,17 +598,17 @@ case ${step} in
595598
esac
596599

597600
walltime="00:10:00"
598-
threads_per_task=1
599601
export is_exclusive=True
600-
tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
602+
threads_per_task=1
603+
tasks_per_node=$(( max_tasks_per_node / 2 ))
601604
;;
602605

603606
"marineanlletkf")
604607
ntasks=16
605608
case ${OCNRES} in
606609
"025")
607610
ntasks=480
608-
memory="96GB"
611+
memory="256GB"
609612
;;
610613
"050")
611614
ntasks=16
@@ -624,18 +627,18 @@ case ${step} in
624627
exit 4
625628
esac
626629

627-
walltime="00:10:00"
628-
threads_per_task=1
630+
walltime="00:30:00"
629631
export is_exclusive=True
630-
tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
632+
threads_per_task=1
633+
tasks_per_node=$(( max_tasks_per_node / 2 ))
631634
;;
632635

633636

634637
"marineanlchkpt")
635638
walltime="00:10:00"
636639
ntasks=1
637640
threads_per_task=1
638-
tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
641+
tasks_per_node=${max_tasks_per_node}
639642
case ${OCNRES} in
640643
"025")
641644
memory="128GB"
@@ -659,7 +662,7 @@ case ${step} in
659662
walltime="00:30:00"
660663
ntasks=${max_tasks_per_node}
661664
threads_per_task=1
662-
tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
665+
tasks_per_node=${max_tasks_per_node}
663666
;;
664667

665668
"ocnanalvrfy")

‎parm/config/gfs/yaml/defaults.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ snowanl:
5353

5454
marineanl:
5555
SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca"
56+
SOCA_ANL_GEOM: "${FIXgfs}/gdas/soca/72x35x25/soca"
5657
SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in oceanprepobs
5758
SOCA_NINNER: 100
5859
JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/soca/marine-jcb-3dfgat.yaml.j2"

‎parm/gdas/soca_bmat_jedi_config.yaml.j2

+19-12
Original file line numberDiff line numberDiff line change
@@ -8,35 +8,42 @@ soca_diagb:
88
rundir: '{{ DATA }}'
99
exe_src: '{{ EXECgfs }}/gdas_soca_diagb.x'
1010
mpi_cmd: '{{ APRUN_MARINEBMAT }}'
11-
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
11+
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
1212
jcb_algo: soca_diagb
13+
soca_chgres:
14+
rundir: '{{ DATA }}'
15+
exe_src: '{{ EXECgfs }}/gdas.x'
16+
mpi_cmd: '{{ APRUN_MARINEBMAT }}'
17+
jedi_args: ['soca', 'convertstate']
18+
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
19+
jcb_algo: soca_chgres
1320
soca_parameters_diffusion_vt:
1421
rundir: '{{ DATA }}'
1522
exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x'
16-
mpi_cmd: '{{ APRUN_MARINEBMAT }}'
17-
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
23+
mpi_cmd: '{{ APRUN_MARINEBMAT }}'
24+
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
1825
jcb_algo: soca_parameters_diffusion_vt
1926
soca_setcorscales:
2027
rundir: '{{ DATA }}'
2128
exe_src: '{{ EXECgfs }}/gdas_soca_setcorscales.x'
22-
mpi_cmd: '{{ APRUN_MARINEBMAT }}'
23-
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
29+
mpi_cmd: '{{ APRUN_MARINEBMAT }}'
30+
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
2431
jcb_algo: soca_setcorscales
2532
soca_parameters_diffusion_hz:
2633
rundir: '{{ DATA }}'
2734
exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x'
28-
mpi_cmd: '{{ APRUN_MARINEBMAT }}'
29-
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
35+
mpi_cmd: '{{ APRUN_MARINEBMAT }}'
36+
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
3037
jcb_algo: soca_parameters_diffusion_hz
3138
soca_ensb:
3239
rundir: '{{ DATA }}'
3340
exe_src: '{{ EXECgfs }}/gdas_ens_handler.x'
34-
mpi_cmd: '{{ APRUN_MARINEBMAT }}'
35-
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
41+
mpi_cmd: '{{ APRUN_MARINEBMAT }}'
42+
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
3643
jcb_algo: soca_ensb
3744
soca_ensweights:
3845
rundir: '{{ DATA }}'
3946
exe_src: '{{ EXECgfs }}/gdas_socahybridweights.x'
40-
mpi_cmd: '{{ APRUN_MARINEBMAT }}'
41-
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
42-
jcb_algo: soca_ensweights
47+
mpi_cmd: '{{ APRUN_MARINEBMAT }}'
48+
jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml'
49+
jcb_algo: soca_ensweights

‎sorc/gdas.cd

Submodule gdas.cd updated 33 files

‎ush/python/pygfs/task/marine_analysis.py

+7-3
Original file line numberDiff line numberDiff line change
@@ -179,9 +179,13 @@ def _prep_scratch_dir(self: Task) -> None:
179179
soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config)
180180
FileHandler(soca_fix_list).sync()
181181

182-
# prepare the MOM6 input.nml
182+
# prepare the deterministic MOM6 input.nml
183183
mdau.prep_input_nml(self.task_config)
184184

185+
# prepare the input.nml for the analysis geometry
186+
mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml",
187+
simple_geom=True, mom_input="./anl_geom/MOM_input")
188+
185189
# stage the soca utility yamls (gridgen, fields and ufo mapping yamls)
186190
logger.info(f"Staging SOCA utility yaml files from {self.task_config.PARMsoca}")
187191
soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config)
@@ -206,7 +210,7 @@ def _prep_variational_yaml(self: Task) -> None:
206210
envconfig_jcb['PARMgfs'] = self.task_config.PARMgfs
207211
envconfig_jcb['NMEM_ENS'] = self.task_config.NMEM_ENS
208212
envconfig_jcb['berror_model'] = 'marine_background_error_static_diffusion'
209-
if self.task_config.NMEM_ENS >= 3:
213+
if self.task_config.NMEM_ENS >= 2:
210214
envconfig_jcb['berror_model'] = 'marine_background_error_hybrid_diffusion_diffusion'
211215
envconfig_jcb['DATA'] = self.task_config.DATA
212216
envconfig_jcb['OPREFIX'] = self.task_config.OPREFIX
@@ -365,7 +369,7 @@ def list_all_files(dir_in, dir_out, wc='*', fh_list=[]):
365369
post_file_list = []
366370

367371
# Make a copy the IAU increment
368-
post_file_list.append([os.path.join(anl_dir, 'inc.nc'),
372+
post_file_list.append([os.path.join(anl_dir, 'ocn.inc.nc'),
369373
os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.ocninc.nc')])
370374

371375
domains = ['ocn', 'ice']

‎ush/python/pygfs/task/marine_bmat.py

+11-2
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ def __init__(self, config):
7474

7575
# Create dictionary of Jedi objects
7676
expected_keys = ['gridgen', 'soca_diagb', 'soca_parameters_diffusion_vt', 'soca_setcorscales',
77-
'soca_parameters_diffusion_hz', 'soca_ensb', 'soca_ensweights']
77+
'soca_parameters_diffusion_hz', 'soca_ensb', 'soca_ensweights', 'soca_chgres']
7878
self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys)
7979

8080
@logit(logger)
@@ -105,9 +105,13 @@ def initialize(self: Task) -> None:
105105
soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config)
106106
FileHandler(soca_fix_list).sync()
107107

108-
# prepare the MOM6 input.nml
108+
# prepare the deterministic MOM6 input.nml
109109
mdau.prep_input_nml(self.task_config)
110110

111+
# prepare the input.nml for the analysis geometry
112+
mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml",
113+
simple_geom=True, mom_input="./anl_geom/MOM_input")
114+
111115
# stage backgrounds
112116
# TODO(G): Check ocean backgrounds dates for consistency
113117
bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config)
@@ -127,6 +131,7 @@ def initialize(self: Task) -> None:
127131
# initialize JEDI applications
128132
self.jedi_dict['gridgen'].initialize(self.task_config)
129133
self.jedi_dict['soca_diagb'].initialize(self.task_config)
134+
self.jedi_dict['soca_chgres'].initialize(self.task_config)
130135
self.jedi_dict['soca_parameters_diffusion_vt'].initialize(self.task_config)
131136
self.jedi_dict['soca_setcorscales'].initialize(self.task_config)
132137
self.jedi_dict['soca_parameters_diffusion_hz'].initialize(self.task_config)
@@ -163,11 +168,15 @@ def execute(self) -> None:
163168
None
164169
"""
165170

171+
# soca grid generation
166172
self.jedi_dict['gridgen'].execute()
167173

168174
# variance partitioning
169175
self.jedi_dict['soca_diagb'].execute()
170176

177+
# Interpolate f009 bkg to analysis geometry
178+
self.jedi_dict['soca_chgres'].execute()
179+
171180
# horizontal diffusion
172181
self.jedi_dict['soca_setcorscales'].execute()
173182
self.jedi_dict['soca_parameters_diffusion_hz'].execute()

‎ush/python/pygfs/utils/marine_da_utils.py

+8-3
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,10 @@ def link_executable(task_config: AttrDict, exe_name: str) -> None:
4545

4646

4747
@logit(logger)
48-
def prep_input_nml(task_config: AttrDict) -> None:
48+
def prep_input_nml(task_config: AttrDict,
49+
output_nml: str = "mom_input.nml",
50+
simple_geom: bool = False,
51+
mom_input: str = "MOM_input") -> None:
4952
"""Prepare the mom_input.nml file
5053
"""
5154
# stage input.nml.j2
@@ -56,9 +59,11 @@ def prep_input_nml(task_config: AttrDict) -> None:
5659
# swap date and stacksize
5760
date_init = [int(s) for s in task_config.MARINE_WINDOW_END.strftime('%Y,%m,%d,%H,%M,%S').split(',')]
5861
input_nml_config = {'domain_stack_size': task_config.DOMAIN_STACK_SIZE,
59-
'date_init': date_init}
62+
'date_init': date_init,
63+
'simple_geom': simple_geom,
64+
'mom_input': mom_input}
6065
jinja_input_nml = jinja.Jinja(mom_input_nml_tmpl, input_nml_config)
61-
jinja_input_nml.save('mom_input.nml')
66+
jinja_input_nml.save(output_nml)
6267

6368

6469
@logit(logger)

‎versions/fix.ver

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ export cpl_ver=20230526
99
export datm_ver=20220805
1010
export gdas_crtm_ver=20220805
1111
export gdas_fv3jedi_ver=20241115
12-
export gdas_soca_ver=20240802
12+
export gdas_soca_ver=20240919
1313
export gdas_gsibec_ver=20240416
1414
export gdas_obs_ver=20240213
1515
export gdas_aero_ver=20240806

0 commit comments

Comments
 (0)