Skip to content

Commit 6b1d621

Browse files
DavidHuber-NOAADavidHuber-NOAA
authored
Port UFS_UTILS to the S4 machine (ufs-community#557)
Update the repository build for S4. New gdas_init utility driver script for S4. Script only invokes the chgres_cube step as S4 does not have access to HPSS. Fixes ufs-community#543.
1 parent 0f5f3cb commit 6b1d621

File tree

5 files changed

+182
-5
lines changed

5 files changed

+182
-5
lines changed

‎cmake/LibMPI.cmake

+5-1
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,11 @@ function (platform_name RETURN_VARIABLE)
154154

155155

156156
set (${RETURN_VARIABLE} "stampede" PARENT_SCOPE)
157-
157+
158+
elseif (SITENAME MATCHES "^s4-submit.ssec.wisc.edu")
159+
160+
set (${RETURN_VARIABLE} "s4" PARENT_SCOPE)
161+
158162
else ()
159163

160164
set (${RETURN_VARIABLE} "unknown" PARENT_SCOPE)

‎fix/link_fixdirs.sh

+6-4
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,20 @@ machine=${2}
99
if [ $# -lt 2 ]; then
1010
set +x
1111
echo '***ERROR*** must specify two arguements: (1) RUN_ENVIR, (2) machine'
12-
echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion )'
12+
echo ' Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion | s4 )'
1313
exit 1
1414
fi
1515

1616
if [ $RUN_ENVIR != emc -a $RUN_ENVIR != nco ]; then
1717
set +x
1818
echo '***ERROR*** unsupported run environment'
19-
echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion )'
19+
echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion | s4 )'
2020
exit 1
2121
fi
22-
if [ $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet -a $machine != orion ]; then
22+
if [ $machine != cray -a $machine != hera -a $machine != dell -a $machine != jet -a $machine != orion -a $machine != s4 ]; then
2323
set +x
2424
echo '***ERROR*** unsupported machine'
25-
echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion )'
25+
echo 'Syntax: link_fv3gfs.sh ( nco | emc ) ( cray | dell | hera | jet | orion | s4 )'
2626
exit 1
2727
fi
2828

@@ -45,6 +45,8 @@ elif [ $machine = "jet" ]; then
4545
FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix"
4646
elif [ $machine = "orion" ]; then
4747
FIX_DIR="/work/noaa/global/glopara/fix"
48+
elif [ $machine = "s4" ]; then
49+
FIX_DIR="/data/prod/glopara/fix"
4850
fi
4951

5052
for dir in fix_am fix_fv3 fix_orog fix_fv3_gmted2010 fix_sfc_climo; do

‎modulefiles/build.s4.intel

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#%Module#####################################################
2+
## Build and run module for S4
3+
#############################################################
4+
5+
module load license_intel/S4
6+
module use /data/prod/hpc-stack/modulefiles/stack
7+
module load hpc/1.1.0
8+
module load hpc-intel/18.0.4
9+
module load hpc-impi/18.0.4
10+
11+
module load bacio/2.4.1
12+
module load g2/3.4.1
13+
module load ip/3.3.3
14+
module load nemsio/2.5.2
15+
module load sp/2.3.3
16+
module load w3nco/2.4.1
17+
module load sfcio/1.4.1
18+
module load sigio/2.3.2
19+
module load wgrib2/2.0.8
20+
21+
module load jasper/2.0.22
22+
module load zlib/1.2.11
23+
module load png/1.6.35
24+
25+
module load hdf5/1.10.6
26+
module load netcdf/4.7.4
27+
module load nccmp/1.8.7.0
28+
module load esmf/8_1_0_beta_snapshot_27

‎sorc/machine-setup.sh

+8
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,14 @@ elif [[ "$(hostname)" =~ "odin" ]]; then
131131
elif [[ -d /work/00315 && -d /scratch/00315 ]] ; then
132132
target=stampede
133133
module purge
134+
elif [[ -d /data/prod ]] ; then
135+
# We are on SSEC S4
136+
if ( ! eval module help > /dev/null 2>&1 ) ; then
137+
echo load the module command 1>&2
138+
source /usr/share/lmod/lmod/init/$__ms_shell
139+
fi
140+
target=s4
141+
module purge
134142
else
135143
echo WARNING: UNKNOWN PLATFORM 1>&2
136144
fi

‎util/gdas_init/driver.s4.sh

+135
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
#!/bin/bash
2+
3+
#---------------------------------------------------------------------
4+
# Driver script for running on S4.
5+
#
6+
# Edit the 'config' file before running.
7+
#---------------------------------------------------------------------
8+
9+
set -x
10+
11+
compiler=${compiler:-"intel"}
12+
source ../../sorc/machine-setup.sh > /dev/null 2>&1
13+
module use ../../modulefiles
14+
module load build.$target.$compiler
15+
module list
16+
17+
# Needed for NDATE utility
18+
module load license_intel/S4
19+
module use /data/prod/hpc-stack/modulefiles/stack
20+
module load hpc/1.1.0
21+
module load hpc-intel/18.0.4
22+
module load hpc-impi/18.0.4
23+
module load prod_util/1.2.2
24+
25+
PROJECT_CODE=star
26+
QUEUE=s4
27+
28+
source config
29+
30+
if [ $EXTRACT_DATA == yes ]; then
31+
32+
echo "HPSS is not accessible from S4! If you wish to run just the chgres portion, set EXTRACT_DATA=NO in the config file and try again."
33+
exit
34+
35+
else # do not extract data.
36+
37+
DEPEND=' '
38+
39+
fi # extract data?
40+
41+
if [ $RUN_CHGRES == yes ]; then
42+
43+
export APRUN=srun
44+
NODES=3
45+
WALLT="0:15:00"
46+
export OMP_NUM_THREADS=1
47+
if [ $CRES_HIRES == 'C768' ] ; then
48+
NODES=5
49+
elif [ $CRES_HIRES == 'C1152' ] ; then
50+
NODES=8
51+
WALLT="0:20:00"
52+
fi
53+
case $gfs_ver in
54+
v12 | v13)
55+
export OMP_NUM_THREADS=4
56+
export OMP_STACKSIZE=1024M
57+
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} --cpus-per-task=$OMP_NUM_THREADS \
58+
-t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
59+
-o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_pre-v14.chgres.sh ${CDUMP}
60+
;;
61+
v14)
62+
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
63+
-o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v14.chgres.sh ${CDUMP}
64+
;;
65+
v15)
66+
if [ "$CDUMP" = "gdas" ]; then
67+
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
68+
-o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v15.chgres.sh ${CDUMP}
69+
else
70+
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
71+
-o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v15.chgres.gfs.sh
72+
fi
73+
;;
74+
v16retro)
75+
if [ "$CDUMP" = "gdas" ] ; then
76+
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
77+
-o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16retro.chgres.sh hires
78+
else
79+
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
80+
-o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16.chgres.sh ${CDUMP}
81+
fi
82+
;;
83+
v16)
84+
sbatch --parsable --ntasks-per-node=6 --nodes=${NODES} -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${CDUMP} \
85+
-o log.${CDUMP} -e log.${CDUMP} ${DEPEND} run_v16.chgres.sh ${CDUMP}
86+
;;
87+
esac
88+
89+
if [ "$CDUMP" = "gdas" ]; then
90+
91+
WALLT="0:15:00"
92+
93+
if [ "$gfs_ver" = "v16retro" ]; then
94+
95+
sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_enkf \
96+
-o log.enkf -e log.enkf ${DEPEND} run_v16retro.chgres.sh enkf
97+
98+
else
99+
100+
MEMBER=1
101+
while [ $MEMBER -le 80 ]; do
102+
if [ $MEMBER -lt 10 ]; then
103+
MEMBER_CH="00${MEMBER}"
104+
else
105+
MEMBER_CH="0${MEMBER}"
106+
fi
107+
case $gfs_ver in
108+
v12 | v13)
109+
export OMP_NUM_THREADS=2
110+
export OMP_STACKSIZE=1024M
111+
sbatch --parsable --ntasks-per-node=12 --nodes=1 --cpus-per-task=$OMP_NUM_THREADS \
112+
-t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \
113+
-o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_pre-v14.chgres.sh ${MEMBER_CH}
114+
;;
115+
v14)
116+
sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \
117+
-o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v14.chgres.sh ${MEMBER_CH}
118+
;;
119+
v15)
120+
sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \
121+
-o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v15.chgres.sh ${MEMBER_CH}
122+
;;
123+
v16)
124+
sbatch --parsable --ntasks-per-node=12 --nodes=1 -t $WALLT -A $PROJECT_CODE -q $QUEUE -J chgres_${MEMBER_CH} \
125+
-o log.${MEMBER_CH} -e log.${MEMBER_CH} ${DEPEND} run_v16.chgres.sh ${MEMBER_CH}
126+
;;
127+
esac
128+
MEMBER=$(( $MEMBER + 1 ))
129+
done
130+
131+
fi # v16 retro?
132+
133+
fi # which CDUMP?
134+
135+
fi # run chgres?

0 commit comments

Comments
 (0)