Issue NOAA-EMC#21 - add phase 3.5 support

- add partition option to setup scripts - remove machine if-blocks from config.base and add variable population to setup_expt*py scripts - add phase 3.5 ppn value to WCOSS_DELL_P3 env and config.resources files
jkhender · Mar 25, 2020 · 08a5354 · 08a5354
1 parent accb6f4
commit 08a5354
Show file tree

Hide file tree

Showing 5 changed files with 131 additions and 72 deletions.
diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env
@@ -14,8 +14,11 @@ fi
 step=$1
 
 # WCOSS_DELL_P3 information
-export npe_node_max=28
 export launcher="mpirun -n"
+export npe_node_max=28
+if [ "$QUEUE" = "dev2" -o "$QUEUE" = "devonprod2" ]; then # WCOSS Dell 3.5
+  export npe_node_max=40
+fi
 
 # Due to ESMF issue, fv3gfs model must run with npe_node_max=24
 if [ $step = "fcst" -o $step = "efcs" ]; then

diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
@@ -12,19 +12,9 @@ export machine="@MACHINE@"
 export RUN_ENVIR="emc"
 
 # Account, queue, etc.
-if [ $machine = "HERA" ]; then
-
-    export ACCOUNT="fv3-cpu"
-    export QUEUE="batch"
-    export QUEUE_ARCH="service"
-
-elif [ $machine = "WCOSS_C" -o $machine = "WCOSS_DELL_P3" ]; then
-
-    export ACCOUNT="GFS-DEV"
-    export QUEUE="dev"
-    export QUEUE_ARCH="dev_transfer"
-
-fi
+export ACCOUNT="@ACCOUNT@"
+export QUEUE="@QUEUE@"
+export QUEUE_ARCH="@QUEUE_ARCH@"
 
 # Project to use in mass store:
 HPSS_PROJECT=emc-global
@@ -41,64 +31,19 @@ export SCRgfs=$HOMEgfs/scripts
 ########################################################################
 
 # GLOBAL static environment parameters
-if [ $machine = "HERA" ]; then
-
-    #export NWPROD="/scratch1/NCEPDEV/global/glopara/nwpara"
-    export DMPDIR="/scratch1/NCEPDEV/global/glopara/dump"
-    export RTMFIX=$CRTM_FIX
-
-elif [ $machine = "WCOSS_C" ]; then
-
-    export NWPROD="/gpfs/hps/nco/ops/nwprod"
-    export DMPDIR="/gpfs/dell3/emc/global/dump"
-    export RTMFIX=$CRTM_FIX
-
-elif [ $machine = "WCOSS_DELL_P3" ]; then
-
-    export NWPROD="/gpfs/dell1/nco/ops/nwprod"
-    export DMPDIR="/gpfs/dell3/emc/global/dump"
-    export RTMFIX=$CRTM_FIX
-fi
-
-
-# Machine specific paths used everywhere
-if [ $machine = "HERA" ]; then
-
-    # USER specific paths
-    export HOMEDIR="/scratch1/NCEPDEV/global/$USER"
-    export STMP="/scratch1/NCEPDEV/stmp2/$USER"
-    export PTMP="/scratch1/NCEPDEV/stmp4/$USER"
-    export NOSCRUB="$HOMEDIR"
-
-    # Base directories for various builds
-    export BASE_GIT="/scratch1/NCEPDEV/global/glopara/git"
-    export BASE_SVN="/scratch1/NCEPDEV/global/glopara/svn"
-
-elif [ $machine = "WCOSS_C" ]; then
-
-    # USER specific paths
-    export HOMEDIR="/gpfs/hps3/emc/global/noscrub/$USER"
-    export STMP="/gpfs/hps2/stmp/$USER"
-    export PTMP="/gpfs/hps2/ptmp/$USER"
-    export NOSCRUB="/gpfs/hps3/emc/global/noscrub/$USER"
-
-    # Base directories for various builds
-    export BASE_GIT="/gpfs/hps3/emc/global/noscrub/emc.glopara/git"
-    export BASE_SVN="/gpfs/hps3/emc/global/noscrub/emc.glopara/svn"
-
-elif [ $machine = "WCOSS_DELL_P3" ]; then
-
-    # USER specific paths
-    export HOMEDIR="/gpfs/dell2/emc/modeling/noscrub/$USER"
-    export STMP="/gpfs/dell3/stmp/$USER"
-    export PTMP="/gpfs/dell3/ptmp/$USER"
-    export NOSCRUB="/gpfs/dell2/emc/modeling/noscrub/$USER"
-
-    # Base directories for various builds
-    export BASE_GIT="/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git"
-    export BASE_SVN="/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git"
-
-fi
+export NWPROD="@NWPROD@"
+export DMPDIR="@DMPDIR@"
+export RTMFIX=$CRTM_FIX
+
+# USER specific paths
+export HOMEDIR="@HOMEDIR@"
+export STMP="@STMP@"
+export PTMP="@PTMP@"
+export NOSCRUB="@NOSCRUB@"
+
+# Base directories for various builds
+export BASE_GIT="@BASE_GIT@"
+export BASE_SVN="@BASE_SVN@"
 
 # Toggle to turn on/off GFS downstream processing.
 export DO_BUFRSND="NO"

diff --git a/parm/config/config.resources b/parm/config/config.resources
@@ -21,6 +21,9 @@ echo "BEGIN: config.resources"
 
 if [[ "$machine" = "WCOSS_DELL_P3" ]]; then
    export npe_node_max=28
+   if [ "$QUEUE" = "dev2" -o "$QUEUE" = "devonprod2" ]; then # WCOSS Dell 3.5
+     export npe_node_max=40
+   fi
 elif [[ "$machine" = "WCOSS_C" ]]; then
    export npe_node_max=24
 elif [[ "$machine" = "JET" ]]; then

diff --git a/ush/rocoto/setup_expt.py b/ush/rocoto/setup_expt.py
@@ -91,6 +91,17 @@ def edit_baseconfig():
                     .replace('@CASECTL@', 'C%d' % resdet) \
                     .replace('@NMEM_ENKF@', '%d' % nens) \
                     .replace('@HOMEgfs@', top) \
+                    .replace('@BASE_GIT@', base_git) \
+                    .replace('@BASE_SVN@', base_svn) \
+                    .replace('@DMPDIR@', dmpdir) \
+                    .replace('@NWPROD@', nwprod) \
+                    .replace('@HOMEDIR@', homedir) \
+                    .replace('@STMP@', stmp) \
+                    .replace('@PTMP@', ptmp) \
+                    .replace('@NOSCRUB@', noscrub) \
+                    .replace('@ACCOUNT@', account) \
+                    .replace('@QUEUE@', queue) \
+                    .replace('@QUEUE_ARCH@', queue_arch) \
                     .replace('@gfs_cyc@', '%d' % gfs_cyc)
                 if expdir is not None:
                     line = line.replace('@EXPDIR@', os.path.dirname(expdir))
@@ -129,6 +140,7 @@ def edit_baseconfig():
     parser.add_argument('--nens', help='number of ensemble members', type=int, required=False, default=20)
     parser.add_argument('--cdump', help='CDUMP to start the experiment', type=str, required=False, default='gdas')
     parser.add_argument('--gfs_cyc', help='GFS cycles to run', type=int, choices=[0, 1, 2, 4], default=1, required=False)
+    parser.add_argument('--partition', help='partition on machine', type=str, required=False, default=None)
 
     args = parser.parse_args()
 
@@ -149,6 +161,48 @@ def edit_baseconfig():
     nens = args.nens
     cdump = args.cdump
     gfs_cyc = args.gfs_cyc
+    partition = args.partition
+
+    # Set machine defaults
+    if machine is 'WCOSS_DELL_P3':
+      base_git = '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git'
+      base_svn = '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git'
+      dmpdir = '/gpfs/dell3/emc/global/dump'
+      nwprod = '/gpfs/dell1/nco/ops/nwprod'
+      homedir = '/gpfs/dell2/emc/modeling/noscrub/$USER'
+      stmp = '/gpfs/dell3/stmp/$USER'
+      ptmp = '/gpfs/dell3/ptmp/$USER'
+      noscrub = '/gpfs/dell2/emc/modeling/noscrub/$USER'
+      account = 'GFS-DEV'
+      queue = 'dev'
+      queue_arch = 'dev_transfer'
+      if partition is not None and partition in ['3p5']:
+        queue = 'dev2'
+        queue_arch = 'dev_transfer2'
+    elif machine is 'WCOSS_C':
+      base_git = '/gpfs/hps3/emc/global/noscrub/emc.glopara/git'
+      base_svn = '/gpfs/hps3/emc/global/noscrub/emc.glopara/svn'
+      dmpdir = '/gpfs/dell3/emc/global/dump'
+      nwprod = '/gpfs/hps/nco/ops/nwprod'
+      homedir = '/gpfs/hps3/emc/global/noscrub/$USER'
+      stmp = '/gpfs/hps2/stmp/$USER'
+      ptmp = '/gpfs/hps2/ptmp/$USER'
+      noscrub = '/gpfs/hps3/emc/global/noscrub/$USER'
+      account = 'GFS-DEV'
+      queue = 'dev'
+      queue_arch = 'dev_transfer'
+    elif machine is 'HERA':
+      base_git = '/scratch1/NCEPDEV/global/glopara/git'
+      base_svn = '/scratch1/NCEPDEV/global/glopara/svn'
+      dmpdir = '/scratch1/NCEPDEV/global/glopara/dump'
+      nwprod = '/scratch1/NCEPDEV/global/glopara/nwpara'
+      homedir = '/scratch1/NCEPDEV/global/$USER'
+      stmp = '/scratch1/NCEPDEV/stmp2/$USER'
+      ptmp = '/scratch1/NCEPDEV/stmp4/$USER'
+      noscrub = '$HOMEDIR'
+      account = 'fv3-cpu'
+      queue = 'batch'
+      queue_arch = 'service'
 
     if args.icsdir is not None and not os.path.exists(icsdir):
         msg = 'Initial conditions do not exist in %s' % icsdir

diff --git a/ush/rocoto/setup_expt_fcstonly.py b/ush/rocoto/setup_expt_fcstonly.py
@@ -65,6 +65,17 @@ def edit_baseconfig():
                     .replace('@EDATE@', edate.strftime('%Y%m%d%H')) \
                     .replace('@CASECTL@', 'C%d' % res) \
                     .replace('@HOMEgfs@', top) \
+                    .replace('@BASE_GIT@', base_git) \
+                    .replace('@BASE_SVN@', base_svn) \
+                    .replace('@DMPDIR@', dmpdir) \
+                    .replace('@NWPROD@', nwprod) \
+                    .replace('@HOMEDIR@', homedir) \
+                    .replace('@STMP@', stmp) \
+                    .replace('@PTMP@', ptmp) \
+                    .replace('@NOSCRUB@', noscrub) \
+                    .replace('@ACCOUNT@', account) \
+                    .replace('@QUEUE@', queue) \
+                    .replace('@QUEUE_ARCH@', queue_arch) \
                     .replace('@gfs_cyc@', '%d' % gfs_cyc)
                 if expdir is not None:
                     line = line.replace('@EXPDIR@', os.path.dirname(expdir))
@@ -99,6 +110,7 @@ def edit_baseconfig():
     parser.add_argument('--edate', help='end date experiment', type=str, required=True)
     parser.add_argument('--configdir', help='full path to directory containing the config files', type=str, required=False, default=None)
     parser.add_argument('--gfs_cyc', help='GFS cycles to run', type=int, choices=[0, 1, 2, 4], default=1, required=False)
+    parser.add_argument('--partition', help='partition on machine', type=str, required=False, default=None)
 
     args = parser.parse_args()
 
@@ -115,6 +127,48 @@ def edit_baseconfig():
     comrot = args.comrot if args.comrot is None else os.path.join(args.comrot, pslot)
     expdir = args.expdir if args.expdir is None else os.path.join(args.expdir, pslot)
     gfs_cyc = args.gfs_cyc
+    partition = args.partition
+
+    # Set machine defaults
+    if machine is 'WCOSS_DELL_P3':
+      base_git = '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git'
+      base_svn = '/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git'
+      dmpdir = '/gpfs/dell3/emc/global/dump'
+      nwprod = '/gpfs/dell1/nco/ops/nwprod'
+      homedir = '/gpfs/dell2/emc/modeling/noscrub/$USER'
+      stmp = '/gpfs/dell3/stmp/$USER'
+      ptmp = '/gpfs/dell3/ptmp/$USER'
+      noscrub = '/gpfs/dell2/emc/modeling/noscrub/$USER'
+      account = 'GFS-DEV'
+      queue = 'dev'
+      queue_arch = 'dev_transfer'
+      if partition is not None and partition in ['3p5']:
+        queue = 'dev2'
+        queue_arch = 'dev_transfer2'
+    elif machine is 'WCOSS_C':
+      base_git = '/gpfs/hps3/emc/global/noscrub/emc.glopara/git'
+      base_svn = '/gpfs/hps3/emc/global/noscrub/emc.glopara/svn'
+      dmpdir = '/gpfs/dell3/emc/global/dump'
+      nwprod = '/gpfs/hps/nco/ops/nwprod'
+      homedir = '/gpfs/hps3/emc/global/noscrub/$USER'
+      stmp = '/gpfs/hps2/stmp/$USER'
+      ptmp = '/gpfs/hps2/ptmp/$USER'
+      noscrub = '/gpfs/hps3/emc/global/noscrub/$USER'
+      account = 'GFS-DEV'
+      queue = 'dev'
+      queue_arch = 'dev_transfer'
+    elif machine is 'HERA':
+      base_git = '/scratch1/NCEPDEV/global/glopara/git'
+      base_svn = '/scratch1/NCEPDEV/global/glopara/svn'
+      dmpdir = '/scratch1/NCEPDEV/global/glopara/dump'
+      nwprod = '/scratch1/NCEPDEV/global/glopara/nwpara'
+      homedir = '/scratch1/NCEPDEV/global/$USER'
+      stmp = '/scratch1/NCEPDEV/stmp2/$USER'
+      ptmp = '/scratch1/NCEPDEV/stmp4/$USER'
+      noscrub = '$HOMEDIR'
+      account = 'fv3-cpu'
+      queue = 'batch'
+      queue_arch = 'service'
 
     # COMROT directory
     create_comrot = True