Merge in latest feature/alt compsetrun branch

tests/.gitignore

@@ -1 +1,2 @@
 /log_*/
+/ecflow_run/

tests/RegressionTests_hera.intel.log

@@ -1,10 +1,10 @@
-Mon Apr 27 02:39:53 UTC 2020
+Wed Apr 29 01:50:12 UTC 2020
 Start Regression test
 
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_cold
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/MEDIATOR
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_mom6_cice_cold_atm_flux
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_mom6_cice_cold_atm_flux
 Checking test 001 cpld_fv3_mom6_cice_cold_atm_flux results ....
  Comparing phyf001.tile1.nc .........OK
  Comparing phyf001.tile2.nc .........OK
@@ -78,7 +78,7 @@ Test 001 cpld_fv3_mom6_cice_cold_atm_flux PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_2d_warm
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_2d_warm/RESTART
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_mom6_cice_2d_atm_flux
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_mom6_cice_2d_atm_flux
 Checking test 002 cpld_fv3_mom6_cice_2d_atm_flux results ....
  Comparing phyf048.tile1.nc .........OK
  Comparing phyf048.tile2.nc .........OK
@@ -157,7 +157,7 @@ Test 002 cpld_fv3_mom6_cice_2d_atm_flux PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_cold_ccpp
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/MEDIATOR_ccpp
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_ccpp_mom6_cice_cold_atm_flux
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_ccpp_mom6_cice_cold_atm_flux
 Checking test 003 cpld_fv3_ccpp_mom6_cice_cold_atm_flux results ....
  Comparing phyf001.tile1.nc .........OK
  Comparing phyf001.tile2.nc .........OK
@@ -231,7 +231,7 @@ Test 003 cpld_fv3_ccpp_mom6_cice_cold_atm_flux PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_cold384_ccpp
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/MEDIATOR384_ccpp
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_ccpp_384_mom6_cice_cold_atm_flux
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_ccpp_384_mom6_cice_cold_atm_flux
 Checking test 004 cpld_fv3_ccpp_384_mom6_cice_cold_atm_flux results ....
  Comparing phyf001.tile1.nc .........OK
  Comparing phyf001.tile2.nc .........OK
@@ -305,7 +305,7 @@ Test 004 cpld_fv3_ccpp_384_mom6_cice_cold_atm_flux PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_2d_warm_ccpp
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_2d_warm_ccpp/RESTART
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_ccpp_mom6_cice_2d_atm_flux
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_ccpp_mom6_cice_2d_atm_flux
 Checking test 005 cpld_fv3_ccpp_mom6_cice_2d_atm_flux results ....
  Comparing phyf048.tile1.nc .........OK
  Comparing phyf048.tile2.nc .........OK
@@ -384,7 +384,7 @@ Test 005 cpld_fv3_ccpp_mom6_cice_2d_atm_flux PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_2d_warm_ccpp384
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_2d_warm_ccpp384/RESTART
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_ccpp_384_mom6_cice_2d_atm_flux
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_ccpp_384_mom6_cice_2d_atm_flux
 Checking test 006 cpld_fv3_ccpp_384_mom6_cice_2d_atm_flux results ....
  Comparing phyf048.tile1.nc .........OK
  Comparing phyf048.tile2.nc .........OK
@@ -463,7 +463,7 @@ Test 006 cpld_fv3_ccpp_384_mom6_cice_2d_atm_flux PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_2d_warm_ccpp
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_2d_warm_ccpp/RESTART
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_ccpp_mom6_cice_2d_2threads
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_ccpp_mom6_cice_2d_2threads
 Checking test 007 cpld_fv3_ccpp_mom6_cice_2d_2threads results ....
  Comparing phyf048.tile1.nc .........OK
  Comparing phyf048.tile2.nc .........OK
@@ -542,7 +542,7 @@ Test 007 cpld_fv3_ccpp_mom6_cice_2d_2threads PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_2d_warm_ccpp
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_2d_warm_ccpp/RESTART
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_ccpp_mom6_cice_2d_decomp
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_ccpp_mom6_cice_2d_decomp
 Checking test 008 cpld_fv3_ccpp_mom6_cice_2d_decomp results ....
  Comparing phyf048.tile1.nc .........OK
  Comparing phyf048.tile2.nc .........OK
@@ -621,7 +621,7 @@ Test 008 cpld_fv3_ccpp_mom6_cice_2d_decomp PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_cold_satmedmf_ccpp
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/MEDIATOR_satmedmf_ccpp
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_ccpp_mom6_cice_cold_satmedmf
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_ccpp_mom6_cice_cold_satmedmf
 Checking test 009 cpld_fv3_ccpp_mom6_cice_cold_satmedmf results ....
  Comparing phyf001.tile1.nc .........OK
  Comparing phyf001.tile2.nc .........OK
@@ -695,7 +695,7 @@ Test 009 cpld_fv3_ccpp_mom6_cice_cold_satmedmf PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_1d_warm_satmedmf_ccpp
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_1d_warm_satmedmf_ccpp/RESTART
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_ccpp_mom6_cice_1d_satmedmf
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_ccpp_mom6_cice_1d_satmedmf
 Checking test 010 cpld_fv3_ccpp_mom6_cice_1d_satmedmf results ....
  Comparing phyf024.tile1.nc .........OK
  Comparing phyf024.tile2.nc .........OK
@@ -774,7 +774,7 @@ Test 010 cpld_fv3_ccpp_mom6_cice_1d_satmedmf PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_cold_bmrt_ccpp
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/MEDIATOR_bmrt_ccpp
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_ccpp_384_mom6_cice_cold_bmark_rt
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_ccpp_384_mom6_cice_cold_bmark_rt
 Checking test 011 cpld_fv3_ccpp_384_mom6_cice_cold_bmark_rt results ....
  Comparing phyf001.tile1.nc .........OK
  Comparing phyf001.tile2.nc .........OK
@@ -848,7 +848,7 @@ Test 011 cpld_fv3_ccpp_384_mom6_cice_cold_bmark_rt PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_1d_bmrt_ccpp
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_1d_bmrt_ccpp/RESTART
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_ccpp_384_mom6_cice_1d_bmark_rt
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_ccpp_384_mom6_cice_1d_bmark_rt
 Checking test 012 cpld_fv3_ccpp_384_mom6_cice_1d_bmark_rt results ....
  Comparing phyf024.tile1.nc .........OK
  Comparing phyf024.tile2.nc .........OK
@@ -927,7 +927,7 @@ Test 012 cpld_fv3_ccpp_384_mom6_cice_1d_bmark_rt PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_cold_bmwav_ccpp
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/MEDIATOR_bmwav_ccpp
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_ccpp_384_mom6_cice_ww3_cold_bmark_rt
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_ccpp_384_mom6_cice_ww3_cold_bmark_rt
 Checking test 013 cpld_fv3_ccpp_384_mom6_cice_ww3_cold_bmark_rt results ....
  Comparing phyf001.tile1.nc .........OK
  Comparing phyf001.tile2.nc .........OK
@@ -1001,7 +1001,7 @@ Test 013 cpld_fv3_ccpp_384_mom6_cice_ww3_cold_bmark_rt PASS
 
 baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_1d_bmwav_ccpp
 mediator baseline dir = /scratch1/NCEPDEV/nems/emc.nemspara/RT/FV3-MOM6-CICE5/develop-20200326/RT-Baselines_1d_bmwav_ccpp/RESTART
-working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_91464/cpld_fv3_ccpp_384_mom6_cice_ww3_1d_bmark_rt
+working dir  = /scratch1/NCEPDEV/stmp2/Minsuk.Ji/S2S_RT/rt_144276/cpld_fv3_ccpp_384_mom6_cice_ww3_1d_bmark_rt
 Checking test 014 cpld_fv3_ccpp_384_mom6_cice_ww3_1d_bmark_rt results ....
  Comparing phyf024.tile1.nc .........OK
  Comparing phyf024.tile2.nc .........OK
@@ -1082,5 +1082,5 @@ Test 014 cpld_fv3_ccpp_384_mom6_cice_ww3_1d_bmark_rt PASS
 
 
 REGRESSION TEST WAS SUCCESSFUL
-Mon Apr 27 10:20:40 UTC 2020
-Elapsed time: 07h:40m:49s. Have a nice day!
+Wed Apr 29 03:40:05 UTC 2020
+Elapsed time: 01h:49m:59s. Have a nice day!

tests/abort_dep_tasks.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+import ecflow
+import re
+
+# this script will work ONLY for standalone nmmb regression test ecflow workflow
+
+class DefsTraverser:
+
+    def __init__(self, defs, ci):
+        assert (isinstance(defs, ecflow.Defs)),"Expected ecflow.Defs as first argument"
+        assert (isinstance(ci, ecflow.Client)),"Expected ecflow.Client as second argument"
+        self.__defs = defs
+        self.__ci = ci
+        self.__suite = None
+
+    def force_abort(self):
+        for suite in self.__defs.suites:
+            self.__suite = suite
+            self.__walk_node(suite)
+
+    def __walk_node(self, node_container):
+        for node in node_container.nodes:
+            if isinstance(node, ecflow.Task):
+                self.__check_task(node)
+            else:
+                self.__walk_node(node)
+
+    def __check_task(self, node):
+        trigger_expr = node.get_trigger()
+        if trigger_expr:
+            tasks = re.findall( r'(\S*) ==', trigger_expr.get_expression())
+            for t in tasks:
+                task = self.__defs.find_abs_node( self.__suite.get_abs_node_path() + "/" + t)
+                if task.get_state() == ecflow.State.aborted:
+                    if node.get_state() != ecflow.State.aborted:
+                        print "Will force aborted state for task", node.get_abs_node_path()
+                        self.__ci.force_state(node.get_abs_node_path(), ecflow.State.aborted)
+
+try:
+    # Create the client. This will read the default environment variables
+    ci = ecflow.Client()
+
+    # Get the node tree suite definition as stored in the server
+    # The definition is retrieved and stored on the variable 'ci'
+    ci.sync_local()
+
+    # access the definition retrieved from the server
+    server_defs = ci.get_defs()
+
+    if server_defs == None :
+        print "The server has no definition"
+        exit(1)
+
+    traverser = DefsTraverser(server_defs, ci)
+    traverser.force_abort()
+
+except RuntimeError, e:
+    print "failed: " + str(e)

tests/default_vars.sh

@@ -113,7 +113,7 @@ COMPILER=${NEMS_COMPILER:-intel}
 if [[ $COMPILER = gnu ]] || [[ $COMPILER = pgi ]]; then
     WLCLK_dflt=30
 else
-    WLCLK_dflt=15
+    WLCLK_dflt=30
 fi
 
 export_fv3 ()

tests/head.h

@@ -0,0 +1,49 @@
+### head.h start
+
+set -e # stop the shell on first error
+set -u # fail when using an undefined variable
+set -x # echo script lines as they are executed
+
+
+# Defines the variables that are needed for any communication with ECF
+export ECF_PORT=%ECF_PORT%    # The server port number
+export ECF_HOST=%ECF_HOST%    # The name of ecf host that issued this task
+export ECF_NAME=%ECF_NAME%    # The name of this current task
+export ECF_PASS=%ECF_PASS%    # A unique password
+export ECF_TRYNO=%ECF_TRYNO%  # Current try number of the task
+export ECF_RID=$$             # record the process id. Also used for zombie detection
+
+# Define the path where to find ecflow_client
+# make sure client and server use the *same* version.
+# Important when there are multiple versions of ecFlow
+#export PATH=....:$PATH
+
+# Tell ecFlow we have started
+ecflow_client --init=$$
+
+
+# Define a error handler
+ERROR() {
+   set +e                      # Clear -e flag, so we don't fail
+   kill $(jobs -p)
+   wait                        # wait for background process to stop
+
+   ecflow_client --ping --host=${ECF_HOST} --port=${ECF_PORT}
+   not_running=$?
+   if [[ $not_running -eq 0 ]]; then
+     export ECF_TIMEOUT=5
+     ecflow_client --abort=trap  # Notify ecFlow that something went wrong, using 'trap' as the reason
+   fi
+   sleep 5
+   trap 0                      # Remove the trap
+   exit 0                      # End the script
+}
+
+
+# Trap any calls to exit and errors caught by the -e flag
+trap ERROR 0
+
+# Trap any signal that may cause the script to fail
+trap '{ echo "$0 Killed by a signal"; ERROR ; }' 1 2 3 4 5 6 7 8   10    12 13    15
+
+### head.h end

tests/rt.sh

@@ -509,7 +509,7 @@ suite ${ECFLOW_SUITE}
     edit ECF_KILL_CMD kill -15 %ECF_RID% > %ECF_JOB%.kill 2>&1
     edit ECF_TRIES 1
     label rundir_root '${RUNDIR_ROOT}'
-    limit max_builds 6
+    limit max_builds 1
     limit max_jobs 30
 EOF
 
@@ -557,6 +557,7 @@ while read -r line; do
       [[ $MACHINES != ' ' && $MACHINES != "${MACHINE_ID}" ]] && continue
       [[ $CREATE_BASELINE == true && $CB != *fv3* ]] && continue
 
+      COMPILE_NR_DEP=${COMPILE_NR}
       (( COMPILE_NR += 1 ))
 
       if [[ $ROCOTO == true ]]; then

tests/rt_utils.sh

@@ -481,7 +481,8 @@ rocoto_create_compile_task() {
   if [[ "Q$APP" != Q ]] ; then
       rocoto_cmd="&PATHRT;/appbuild.sh &PATHTR;/FV3 $APP $COMPILE_NR"
   else
-      rocoto_cmd="&PATHRT;/compile_cmake.sh &PATHTR; $MACHINE_ID \"${NEMS_VER}\" $COMPILE_NR"
+      #rocoto_cmd="&PATHRT;/compile_cmake.sh &PATHTR; $MACHINE_ID \"${NEMS_VER}\" $COMPILE_NR"
+      rocoto_cmd="&PATHRT;/compile.sh ${NEMS_VER} $COMPILE_NR"
   fi
 
   NATIVE=""
@@ -503,8 +504,10 @@ rocoto_create_compile_task() {
     BUILD_WALLTIME="01:00:00"
   fi
 
+  if [[ ${COMPILE_NR_DEP} -gt 0 ]]; then
   cat << EOF >> $ROCOTO_XML
   <task name="compile_${COMPILE_NR}" maxtries="3">
+    <dependency> <taskdep task="compile_${COMPILE_NR_DEP}"/></dependency>
     <command>$rocoto_cmd</command>
     <jobname>compile_${COMPILE_NR}</jobname>
     <account>${ACCNR}</account>
@@ -516,6 +519,21 @@ rocoto_create_compile_task() {
     ${NATIVE}
   </task>
 EOF
+  else
+  cat << EOF >> $ROCOTO_XML
+  <task name="compile_${COMPILE_NR}" maxtries="3">
+    <command>$rocoto_cmd</command>
+    <jobname>compile_${COMPILE_NR}</jobname>
+    <account>${ACCNR}</account>
+    <queue>${COMPILE_QUEUE}</queue>
+    <partition>${PARTITION}</partition>
+    <cores>${BUILD_CORES}</cores>
+    <walltime>${BUILD_WALLTIME}</walltime>
+    <join>&LOG;/compile_${COMPILE_NR}.log</join>
+    ${NATIVE}
+  </task>
+EOF
+  fi
 }
 
 
@@ -592,7 +610,8 @@ ecflow_create_compile_task() {
   if [[ "Q$APP" != Q ]] ; then
       ecflow_cmd="$PATHRT/appbuild.sh ${PATHTR}/FV3 $APP $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1"
   else
-      ecflow_cmd="$PATHRT/compile_cmake.sh ${PATHTR} $MACHINE_ID \"${NEMS_VER}\" $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1"
+      #ecflow_cmd="$PATHRT/compile_cmake.sh ${PATHTR} $MACHINE_ID \"${NEMS_VER}\" $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1"
+      ecflow_cmd="$PATHRT/compile.sh ${NEMS_VER} $COMPILE_NR > ${LOG_DIR}/compile_${COMPILE_NR}.log 2>&1"
   fi
 
   cat << EOF > ${ECFLOW_RUN}/${ECFLOW_SUITE}/compile_${COMPILE_NR}.ecf

tests/tail.h

@@ -0,0 +1,13 @@
+
+### tail.h start
+wait                      # wait for background process to stop
+
+ecflow_client --ping --host=${ECF_HOST} --port=${ECF_PORT}
+not_running=$?
+if [[ $not_running -eq 0 ]]; then
+  ecflow_client --complete  # Notify ecFlow of a normal end
+fi
+
+trap 0                    # Remove all traps
+exit 0                    # End the shell
+### tail.h end