diff --git a/parm/config/gefs/config.base.emc.dyn b/parm/config/gefs/config.base.emc.dyn index ff2fe3377b..051a2188c3 100644 --- a/parm/config/gefs/config.base.emc.dyn +++ b/parm/config/gefs/config.base.emc.dyn @@ -144,7 +144,7 @@ export DO_OCN="NO" export DO_ICE="NO" export DO_AERO="NO" export WAVE_CDUMP="" # When to include wave suite: gdas, gfs, or both -export DOBNDPNT_WAVE="NO" +export DOBNDPNT_WAVE="NO" # The GEFS buoys file does not currently have any boundary points export FRAC_GRID=".true." # Set operational resolution diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 36b70aecb8..a4ae76d7fb 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -68,6 +68,15 @@ esac export npe_node_max case ${step} in + + "stage_ic") + export wtime_stage_ic="00:15:00" + export npe_stage_ic=1 + export npe_node_stage_ic=1 + export nth_stage_ic=1 + export is_exclusive=True + ;; + "waveinit") export wtime_waveinit="00:10:00" export npe_waveinit=12 @@ -77,25 +86,10 @@ case ${step} in export memory_waveinit="2GB" ;; - "wavepostsbs") - export wtime_wavepostsbs="00:20:00" - export wtime_wavepostsbs_gfs="03:00:00" - export npe_wavepostsbs=8 - export nth_wavepostsbs=1 - export npe_node_wavepostsbs=$(( npe_node_max / nth_wavepostsbs )) - export NTASKS=${npe_wavepostsbs} - export memory_wavepostsbs="10GB" - export memory_wavepostsbs_gfs="10GB" - ;; - "fcst" | "efcs") export is_exclusive=True - if [[ "${step}" == "fcst" ]]; then - _CDUMP_LIST=${CDUMP:-"gdas gfs"} - elif [[ "${step}" == "efcs" ]]; then - _CDUMP_LIST=${CDUMP:-"enkfgdas enkfgfs"} - fi + _CDUMP_LIST=${CDUMP:-"gdas gfs"} # During workflow creation, we need resources for all CDUMPs and CDUMP is undefined for _CDUMP in ${_CDUMP_LIST}; do @@ -224,11 +218,39 @@ case ${step} in export is_exclusive=True ;; - "stage_ic") - export wtime_stage_ic="00:15:00" - export npe_stage_ic=1 - export npe_node_stage_ic=1 - export nth_stage_ic=1 + "wavepostsbs") + export wtime_wavepostsbs="03:00:00" + export npe_wavepostsbs=1 + export nth_wavepostsbs=1 + export npe_node_wavepostsbs=$(( npe_node_max / nth_wavepostsbs )) + export NTASKS=${npe_wavepostsbs} + export memory_wavepostsbs="10GB" + ;; + + "wavepostbndpnt") + export wtime_wavepostbndpnt="01:00:00" + export npe_wavepostbndpnt=240 + export nth_wavepostbndpnt=1 + export npe_node_wavepostbndpnt=$(( npe_node_max / nth_wavepostbndpnt )) + export NTASKS=${npe_wavepostbndpnt} + export is_exclusive=True + ;; + + "wavepostbndpntbll") + export wtime_wavepostbndpntbll="01:00:00" + export npe_wavepostbndpntbll=448 + export nth_wavepostbndpntbll=1 + export npe_node_wavepostbndpntbll=$(( npe_node_max / nth_wavepostbndpntbll )) + export NTASKS=${npe_wavepostbndpntbll} + export is_exclusive=True + ;; + + "wavepostpnt") + export wtime_wavepostpnt="04:00:00" + export npe_wavepostpnt=200 + export nth_wavepostpnt=1 + export npe_node_wavepostpnt=$(( npe_node_max / nth_wavepostpnt )) + export NTASKS=${npe_wavepostpnt} export is_exclusive=True ;; @@ -239,4 +261,4 @@ case ${step} in esac -echo "END: config.resources" \ No newline at end of file +echo "END: config.resources" diff --git a/parm/config/gefs/config.wavepostbndpnt b/parm/config/gefs/config.wavepostbndpnt new file mode 100644 index 0000000000..412c5fb42a --- /dev/null +++ b/parm/config/gefs/config.wavepostbndpnt @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +########## config.wavepostbndpnt ########## +# Wave steps specific + +echo "BEGIN: config.wavepostbndpnt" + +# Get task specific resources +source "${EXPDIR}/config.resources" wavepostbndpnt + +echo "END: config.wavepostbndpnt" diff --git a/parm/config/gefs/config.wavepostbndpntbll b/parm/config/gefs/config.wavepostbndpntbll new file mode 100644 index 0000000000..6695ab0f84 --- /dev/null +++ b/parm/config/gefs/config.wavepostbndpntbll @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +########## config.wavepostbndpntbll ########## +# Wave steps specific + +echo "BEGIN: config.wavepostbndpntbll" + +# Get task specific resources +source "${EXPDIR}/config.resources" wavepostbndpntbll + +echo "END: config.wavepostbndpntbll" diff --git a/parm/config/gefs/config.wavepostpnt b/parm/config/gefs/config.wavepostpnt new file mode 100644 index 0000000000..e87237da82 --- /dev/null +++ b/parm/config/gefs/config.wavepostpnt @@ -0,0 +1,11 @@ +#! /usr/bin/env bash + +########## config.wavepostpnt ########## +# Wave steps specific + +echo "BEGIN: config.wavepostpnt" + +# Get task specific resources +source "${EXPDIR}/config.resources" wavepostpnt + +echo "END: config.wavepostpnt" diff --git a/parm/config/gefs/config.wavepostsbs b/parm/config/gefs/config.wavepostsbs new file mode 100644 index 0000000000..b3c5902e3c --- /dev/null +++ b/parm/config/gefs/config.wavepostsbs @@ -0,0 +1,28 @@ +#! /usr/bin/env bash + +########## config.wavepostsbs ########## +# Wave steps specific + +echo "BEGIN: config.wavepostsbs" + +# Get task specific resources +source "${EXPDIR}/config.resources" wavepostsbs + +# Subgrid info for grib2 encoding +export WAV_SUBGRBSRC="" +export WAV_SUBGRB="" + +# Options for point output (switch on/off boundary point output) +export DOIBP_WAV='NO' # Input boundary points +export DOFLD_WAV='YES' # Field data +export DOPNT_WAV='YES' # Station data +export DOGRB_WAV='YES' # Create grib2 files +if [[ -n "${waveinterpGRD}" ]]; then + export DOGRI_WAV='YES' # Create interpolated grids +else + export DOGRI_WAV='NO' # Do not create interpolated grids +fi +export DOSPC_WAV='YES' # Spectral post +export DOBLL_WAV='YES' # Bulletin post + +echo "END: config.wavepostsbs" diff --git a/parm/config/gfs/config.wavepostbndpnt b/parm/config/gfs/config.wavepostbndpnt index dfeddc79b2..412c5fb42a 100644 --- a/parm/config/gfs/config.wavepostbndpnt +++ b/parm/config/gfs/config.wavepostbndpnt @@ -6,6 +6,6 @@ echo "BEGIN: config.wavepostbndpnt" # Get task specific resources -. $EXPDIR/config.resources wavepostbndpnt +source "${EXPDIR}/config.resources" wavepostbndpnt echo "END: config.wavepostbndpnt" diff --git a/parm/config/gfs/config.wavepostbndpntbll b/parm/config/gfs/config.wavepostbndpntbll index bb7224cc70..6695ab0f84 100644 --- a/parm/config/gfs/config.wavepostbndpntbll +++ b/parm/config/gfs/config.wavepostbndpntbll @@ -6,6 +6,6 @@ echo "BEGIN: config.wavepostbndpntbll" # Get task specific resources -. $EXPDIR/config.resources wavepostbndpntbll +source "${EXPDIR}/config.resources" wavepostbndpntbll echo "END: config.wavepostbndpntbll" diff --git a/parm/config/gfs/config.wavepostpnt b/parm/config/gfs/config.wavepostpnt index 8befb91760..e87237da82 100644 --- a/parm/config/gfs/config.wavepostpnt +++ b/parm/config/gfs/config.wavepostpnt @@ -6,6 +6,6 @@ echo "BEGIN: config.wavepostpnt" # Get task specific resources -. $EXPDIR/config.resources wavepostpnt +source "${EXPDIR}/config.resources" wavepostpnt echo "END: config.wavepostpnt" diff --git a/parm/config/gfs/config.wavepostsbs b/parm/config/gfs/config.wavepostsbs index 8e74aae069..b3c5902e3c 100644 --- a/parm/config/gfs/config.wavepostsbs +++ b/parm/config/gfs/config.wavepostsbs @@ -6,7 +6,7 @@ echo "BEGIN: config.wavepostsbs" # Get task specific resources -. $EXPDIR/config.resources wavepostsbs +source "${EXPDIR}/config.resources" wavepostsbs # Subgrid info for grib2 encoding export WAV_SUBGRBSRC="" diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh index a7aa957564..c085c48f30 100755 --- a/scripts/exgfs_wave_post_pnt.sh +++ b/scripts/exgfs_wave_post_pnt.sh @@ -156,7 +156,11 @@ source "$HOMEgfs/ush/preamble.sh" cp -f $PARMwave/wave_${NET}.buoys buoy.loc.temp if [ "$DOBNDPNT_WAV" = YES ]; then #only do boundary points - sed -n '/^\$.*/!p' buoy.loc.temp | grep IBP > buoy.loc + sed -n '/^\$.*/!p' buoy.loc.temp | grep IBP > buoy.loc || { + echo "WARNING: No boundary points found in buoy file ${PARMwave}/wave_${NET}.buoys" + echo " Ending job without doing anything." + exit 0 + } else #exclude boundary points sed -n '/^\$.*/!p' buoy.loc.temp | grep -v IBP > buoy.loc diff --git a/workflow/applications/gefs.py b/workflow/applications/gefs.py index 9e8bb5c67e..1073397a08 100644 --- a/workflow/applications/gefs.py +++ b/workflow/applications/gefs.py @@ -20,7 +20,9 @@ def _get_app_configs(self): configs += ['efcs'] if self.do_wave: - configs += ['waveinit'] + configs += ['waveinit', 'wavepostsbs', 'wavepostpnt'] + if self.do_wave_bnd: + configs += ['wavepostbndpnt', 'wavepostbndpntbll'] return configs @@ -47,4 +49,10 @@ def get_task_names(self): tasks += ['atmprod'] + if self.do_wave: + tasks += ['wavepostsbs'] + if self.do_wave_bnd: + tasks += ['wavepostbndpnt', 'wavepostbndpntbll'] + tasks += ['wavepostpnt'] + return {f"{self._base['CDUMP']}": tasks} diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py index a72753eb90..3eb249dc76 100644 --- a/workflow/rocoto/gefs_tasks.py +++ b/workflow/rocoto/gefs_tasks.py @@ -75,7 +75,7 @@ def stage_ic(self): def waveinit(self): resources = self.get_resource('waveinit') - task_name = f'waveinit' + task_name = f'wave_init' task_dict = {'task_name': task_name, 'resources': resources, 'envars': self.envars, @@ -90,14 +90,12 @@ def waveinit(self): return task def fcst(self): - - # TODO: Add real dependencies dependencies = [] dep_dict = {'type': 'task', 'name': f'stage_ic'} dependencies.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_wave: - dep_dict = {'type': 'task', 'name': f'waveinit'} + dep_dict = {'type': 'task', 'name': f'wave_init'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) @@ -124,7 +122,7 @@ def efcs(self): dependencies.append(rocoto.add_dependency(dep_dict)) if self.app_config.do_wave: - dep_dict = {'type': 'task', 'name': f'waveinit'} + dep_dict = {'type': 'task', 'name': f'wave_init'} dependencies.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=dependencies) @@ -206,3 +204,160 @@ def atmprod(self): task = rocoto.create_task(member_metatask_dict) return task + + def wavepostsbs(self): + deps = [] + for wave_grid in self._configs['wavepostsbs']['waveGRD'].split(): + wave_hist_path = self._template_to_rocoto_cycstring(self._base["COM_WAVE_HISTORY_TMPL"], {'MEMDIR': 'mem#member#'}) + data = f'{wave_hist_path}/gefswave.out_grd.{wave_grid}.@Y@m@d.@H0000' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + wave_post_envars = self.envars.copy() + postenvar_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + } + for key, value in postenvar_dict.items(): + wave_post_envars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('wavepostsbs') + + task_name = f'wave_post_grid_mem#member#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': wave_post_envars, + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostsbs.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': 'wave_post_grid', + 'task_dict': task_dict, + 'var_dict': member_var_dict + } + + task = rocoto.create_task(member_metatask_dict) + + return task + + def wavepostbndpnt(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'fcst_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + wave_post_bndpnt_envars = self.envars.copy() + postenvar_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + } + for key, value in postenvar_dict.items(): + wave_post_bndpnt_envars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('wavepostbndpnt') + task_name = f'wave_post_bndpnt_mem#member#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': wave_post_bndpnt_envars, + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpnt.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': 'wave_post_bndpnt', + 'task_dict': task_dict, + 'var_dict': member_var_dict + } + + task = rocoto.create_task(member_metatask_dict) + + return task + + def wavepostbndpntbll(self): + deps = [] + atmos_hist_path = self._template_to_rocoto_cycstring(self._base["COM_ATMOS_HISTORY_TMPL"], {'MEMDIR': 'mem#member#'}) + # Is there any reason this is 180? + data = f'{atmos_hist_path}/{self.cdump}.t@Hz.atm.logf180.txt' + dep_dict = {'type': 'data', 'data': data} + deps.append(rocoto.add_dependency(dep_dict)) + + dep_dict = {'type': 'task', 'name': f'fcst_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='or', dep=deps) + + wave_post_bndpnt_bull_envars = self.envars.copy() + postenvar_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + } + for key, value in postenvar_dict.items(): + wave_post_bndpnt_bull_envars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('wavepostbndpntbll') + task_name = f'wave_post_bndpnt_bull_mem#member#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': wave_post_bndpnt_bull_envars, + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostbndpntbll.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': 'wave_post_bndpnt_bull', + 'task_dict': task_dict, + 'var_dict': member_var_dict + } + + task = rocoto.create_task(member_metatask_dict) + + return task + + def wavepostpnt(self): + deps = [] + dep_dict = {'type': 'task', 'name': f'fcst_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_wave_bnd: + dep_dict = {'type': 'task', 'name': f'wave_post_bndpnt_bull_mem#member#'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + wave_post_pnt_envars = self.envars.copy() + postenvar_dict = {'ENSMEM': '#member#', + 'MEMDIR': 'mem#member#', + } + for key, value in postenvar_dict.items(): + wave_post_pnt_envars.append(rocoto.create_envar(name=key, value=str(value))) + + resources = self.get_resource('wavepostpnt') + task_name = f'wave_post_pnt_mem#member#' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': wave_post_pnt_envars, + 'cycledef': 'gefs', + 'command': f'{self.HOMEgfs}/jobs/rocoto/wavepostpnt.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])} + member_metatask_dict = {'task_name': 'wave_post_pnt', + 'task_dict': task_dict, + 'var_dict': member_var_dict + } + + task = rocoto.create_task(member_metatask_dict) + + return task