Merge pull request #842 from buildtesters/executor_settings

add 'executors' property for multiple executor definition

.gitignore

@@ -7,6 +7,7 @@ __pycache__/
 buildtest_framework.egg-info/
 dist/
 build/
+var/
 
 # xml files generated for buildtest cdash
 *.xml

buildtest/buildsystem/base.py

@@ -31,6 +31,7 @@
 from buildtest.utils.file import create_dir, read_file, write_file
 from buildtest.utils.shell import Shell
 from buildtest.utils.timer import Timer
+from buildtest.utils.tools import deep_get
 
 
 class BuilderBase(ABC):
@@ -123,6 +124,8 @@ def __init__(self, name, recipe, buildspec, executor, buildexecutor, testdir):
         elif self.shell.name in ["python"]:
             self.shell_type = "python"
 
+        self.sched_init()
+
     def _set_metadata_values(self):
         """This method sets self.metadata that contains metadata for each builder object."""
         self.metadata["name"] = self.name
@@ -183,8 +186,9 @@ def _set_metadata_values(self):
         # used to store job metrics for given JobID from batch scheduler
         self.metadata["job"] = None
         # Generate a unique id for the build based on key and unique string
-        self.metadata["full_id"] = self._generate_unique_id()
-        self.metadata["id"] = self.metadata["full_id"][:8]
+        self.test_uid = self._generate_unique_id()
+        self.metadata["full_id"] = self.test_uid
+        self.metadata["id"] = self.test_uid[:8]
 
     def _generate_unique_id(self):
         """Generate a unique build id using ``uuid.uuid4()``."""
@@ -329,12 +333,13 @@ def _build_setup(self):
         """
 
         create_dir(self.testdir)
-        num_content = len(os.listdir(self.testdir))
+
+        # num_content = len(os.listdir(self.testdir))
         # the testid is incremented for every run, this can be done by getting
         # length of all files in testdir and creating a directory. Subsequent
         # runs will increment this counter
 
-        self.test_root = os.path.join(self.testdir, str(num_content))
+        self.test_root = os.path.join(self.testdir, self.test_uid[:8])
 
         create_dir(self.test_root)
         self.metadata["testroot"] = self.test_root
@@ -474,53 +479,94 @@ def _write_test(self):
             os.path.join(self.test_root, os.path.basename(self.testpath)),
         )
 
-    def _get_scheduler_directives(
-        self, bsub=None, sbatch=None, cobalt=None, pbs=None, batch=None
-    ):
-        """Get Scheduler Directives for LSF, Slurm, PBS or Cobalt if we are processing
-        test with one of the executor types. This method will return a list
-        of string containing scheduler directives generally found at top of script.
-        If test is local executor we return an empty list
-
-        :param bsub: bsub property from buildspec
-        :param sbatch: sbatch property from buildspec
-        :param cobalt: cobalt property  from buildspec
-        :param pbs: pbs property  from buildspec
-        :param batch: batch property from buildspec
-        """
+    def sched_init(self):
+        """This method will resolve scheduler fields: 'sbatch', 'pbs', 'bsub', 'cobalt'"""
+        self.sbatch = deep_get(
+            self.recipe, "executors", self.executor, "sbatch"
+        ) or self.recipe.get("sbatch")
+        self.lsf = deep_get(
+            self.recipe, "executors", self.executor, "bsub"
+        ) or self.recipe.get("bsub")
+        self.pbs = deep_get(
+            self.recipe, "executors", self.executor, "pbs"
+        ) or self.recipe.get("pbs")
+        self.cobalt = deep_get(
+            self.recipe, "executors", self.executor, "cobalt"
+        ) or self.recipe.get("cobalt")
+        self.batch = self.recipe.get("batch")
+
+        self.burstbuffer = self.recipe.get("BB") or deep_get(
+            self.recipe, "executors", self.executor, "BB"
+        )
+        self.datawarp = self.recipe.get("DW") or deep_get(
+            self.recipe, "executors", self.executor, "DW"
+        )
+
+    def get_slurm_directives(self):
+        """Get #SBATCH lines based on ``sbatch`` property"""
+        jobscript = SlurmBatchScript(sbatch=self.sbatch, batch=self.batch)
+        lines = jobscript.get_headers()
+        lines += [f"#SBATCH --job-name={self.name}"]
+        lines += [f"#SBATCH --output={self.name}.out"]
+        lines += [f"#SBATCH --output={self.name}.err"]
+
+        return lines
+
+    def get_lsf_directives(self):
+        """Get #BSUB lines based on ``bsub`` property"""
+        jobscript = LSFBatchScript(bsub=self.bsub, batch=self.batch)
+        lines = jobscript.get_headers()
+        lines += [f"#BSUB -J {self.name}"]
+        lines += [f"#BSUB -o {self.name}.out"]
+        lines += [f"#BSUB -e {self.name}.err"]
+
+        return lines
+
+    def get_pbs_directives(self):
+        """Get #PBS lines based on ``pbs`` property"""
+        jobscript = PBSBatchScript(pbs=self.pbs, batch=self.batch)
+        lines = jobscript.get_headers()
+        lines += [f"#PBS -N {self.name}"]
+
+        return lines
+
+    def get_cobalt_directives(self):
+        """Get #COBALT lines based on ``cobalt`` property"""
+        jobscript = CobaltBatchScript(cobalt=self.cobalt, batch=self.batch)
+        lines = jobscript.get_headers()
+        lines += [f"#COBALT --jobname {self.name}"]
 
+        return lines
+
+    def get_job_directives(self):
+        """This method returns a list of lines containing the scheduler directives"""
         lines = []
-        # print(self.buildexecutor.is_slurm(self.executor_type), self.executor_type, sbatch, batch)
-
-        if bsub:
-            script = LSFBatchScript(batch=batch, bsub=bsub)
-
-            lines += script.get_headers()
-            lines += [f"#BSUB -J {self.name}"]
-            lines += [f"#BSUB -o {self.name}.out"]
-            lines += [f"#BSUB -e {self.name}.err"]
-            return lines
-
-        if sbatch:
-
-            script = SlurmBatchScript(batch=batch, sbatch=sbatch)
-            lines += script.get_headers()
-            lines += [f"#SBATCH --job-name={self.name}"]
-            lines += [f"#SBATCH --output={self.name}.out"]
-            lines += [f"#SBATCH --error={self.name}.err"]
-            return lines
-
-        if cobalt:
-            script = CobaltBatchScript(batch=batch, cobalt=cobalt)
-            lines += script.get_headers()
-            lines += [f"#COBALT --jobname {self.name}"]
-            return lines
-
-        if pbs:
-            script = PBSBatchScript(batch=batch, pbs=pbs)
-            lines += script.get_headers()
-            lines += [f"#PBS -N {self.name}"]
-            return lines
+
+        if self.sbatch:
+            sbatch_lines = self.get_slurm_directives()
+            lines.append("####### START OF SCHEDULER DIRECTIVES #######")
+            lines += sbatch_lines
+            lines.append("####### END OF SCHEDULER DIRECTIVES   #######")
+
+        if self.lsf:
+            bsub_lines = self.get_lsf_directives()
+            lines.append("####### START OF SCHEDULER DIRECTIVES #######")
+            lines += bsub_lines
+            lines.append("####### END OF SCHEDULER DIRECTIVES   #######")
+
+        if self.pbs:
+            pbs_lines = self.get_pbs_directives()
+            lines.append("####### START OF SCHEDULER DIRECTIVES #######")
+            lines += pbs_lines
+            lines.append("####### END OF SCHEDULER DIRECTIVES   #######")
+
+        if self.cobalt:
+            cobalt_lines = self.get_cobalt_directives()
+            lines.append("####### START OF SCHEDULER DIRECTIVES #######")
+            lines += cobalt_lines
+            lines.append("####### END OF SCHEDULER DIRECTIVES   #######")
+
+        return lines
 
     def _get_burst_buffer(self, burstbuffer):
         """Get Burst Buffer directives (#BB) lines specified by BB property
@@ -535,9 +581,11 @@ def _get_burst_buffer(self, burstbuffer):
             return
 
         lines = []
+        lines.append("####### START OF BURST BUFFER DIRECTIVES #######")
         for arg in burstbuffer:
             lines += [f"#BB {arg} "]
 
+        lines.append("####### END OF BURST BUFFER DIRECTIVES   #######")
         return lines
 
     def _get_data_warp(self, datawarp):
@@ -553,9 +601,11 @@ def _get_data_warp(self, datawarp):
             return
 
         lines = []
+        lines.append("####### START OF DATAWARP DIRECTIVES #######")
         for arg in datawarp:
             lines += [f"#DW {arg}"]
 
+        lines.append("####### END OF DATAWARP DIRECTIVES   #######")
         return lines
 
     def _set_execute_perm(self, fname):

buildtest/buildsystem/compilerbuilder.py

@@ -177,18 +177,16 @@ def generate_script(self):
                 or deep_get(self.compiler_section, "default", "all", batch)
             )
 
-        batch_directives_lines = self._get_scheduler_directives(
-            bsub=batch_dict["bsub"],
-            sbatch=batch_dict["sbatch"],
-            cobalt=batch_dict["cobalt"],
-            pbs=batch_dict["pbs"],
-            batch=batch_dict["batch"],
-        )
-
-        if batch_directives_lines:
-            lines.append("### START OF SCHEDULER DIRECTIVES ###")
-            lines += batch_directives_lines
-            lines.append("### END OF SCHEDULER DIRECTIVES   ###")
+        # setting these values override values from Builder.sched_init() method
+        self.sbatch = batch_dict["sbatch"]
+        self.lsf = batch_dict["bsub"]
+        self.pbs = batch_dict["pbs"]
+        self.cobalt = batch_dict["cobalt"]
+        self.batch = batch_dict["batch"]
+
+        sched_lines = self.get_job_directives()
+        if sched_lines:
+            lines += sched_lines
 
         # get cray burst buffer (BB) and datawarp (DW) fields in order of precedence.
         for name in ["BB", "DW"]:

buildtest/buildsystem/scriptbuilder.py

@@ -3,6 +3,7 @@
 
 from buildtest.buildsystem.base import BuilderBase
 from buildtest.utils.file import write_file
+from buildtest.utils.tools import deep_get
 
 
 class ScriptBuilder(BuilderBase):
@@ -50,32 +51,21 @@ def generate_script(self):
         if self.shell.name == "python":
             lines = [self.default_shell.shebang]
 
-        batch_directives_lines = self._get_scheduler_directives(
-            bsub=self.recipe.get("bsub"),
-            sbatch=self.recipe.get("sbatch"),
-            cobalt=self.recipe.get("cobalt"),
-            pbs=self.recipe.get("pbs"),
-            batch=self.recipe.get("batch"),
-        )
-        if batch_directives_lines:
-            lines.append("####### START OF SCHEDULER DIRECTIVES #######")
-            lines += batch_directives_lines
-            lines.append("####### END OF SCHEDULER DIRECTIVES   #######")
-            lines.append("\n")
-
-        burst_buffer_lines = self._get_burst_buffer(self.recipe.get("BB"))
-        if burst_buffer_lines:
-            lines.append("####### START OF BURST BUFFER DIRECTIVES #######")
-            lines += burst_buffer_lines
-            lines.append("####### END OF BURST BUFFER DIRECTIVES   #######")
-            lines.append("\n")
-
-        data_warp_lines = self._get_data_warp(self.recipe.get("DW"))
-        if data_warp_lines:
-            lines.append("####### START OF DATAWARP DIRECTIVES #######")
-            lines += data_warp_lines
-            lines.append("####### END OF DATAWARP DIRECTIVES   #######")
-            lines.append("\n")
+        sched_lines = self.get_job_directives()
+        if sched_lines:
+            lines += sched_lines
+
+        if self.burstbuffer:
+
+            burst_buffer_lines = self._get_burst_buffer(self.burstbuffer)
+            if burst_buffer_lines:
+                lines += burst_buffer_lines
+
+        if self.datawarp:
+            data_warp_lines = self._get_data_warp(self.datawarp)
+
+            if data_warp_lines:
+                lines += data_warp_lines
 
         # for python scripts we generate python script and return lines
         if self.shell.name == "python":
@@ -88,13 +78,23 @@ def generate_script(self):
 
         # Add environment variables
         env_lines = self._get_environment(self.recipe.get("env"))
+        # Add variables
+        var_lines = self._get_variables(self.recipe.get("vars"))
+
+        # if environment section defined within 'executors' field then read this instead
+        if deep_get(self.recipe, "executors", self.executor, "env"):
+            env_lines = self._get_environment(
+                self.recipe["executors"][self.executor]["env"]
+            )
+
+        if deep_get(self.recipe, "executors", self.executor, "vars"):
+            var_lines = self._get_environment(
+                self.recipe["executors"][self.executor]["vars"]
+            )
 
         if env_lines:
             lines += env_lines
 
-        # Add variables
-        var_lines = self._get_variables(self.recipe.get("vars"))
-
         if var_lines:
             lines += var_lines
 

buildtest/buildsystem/spack.py

@@ -8,6 +8,7 @@
 from buildtest.buildsystem.base import BuilderBase
 from buildtest.exceptions import BuildTestError
 from buildtest.utils.file import resolve_path
+from buildtest.utils.tools import deep_get
 
 
 class SpackBuilder(BuilderBase):
@@ -38,27 +39,37 @@ def generate_script(self):
 
         lines = ["#!/bin/bash"]
 
-        batch_directives_lines = self._get_scheduler_directives(
-            bsub=self.recipe.get("bsub"),
-            sbatch=self.recipe.get("sbatch"),
-            cobalt=self.recipe.get("cobalt"),
-            pbs=self.recipe.get("pbs"),
-            batch=self.recipe.get("batch"),
-        )
+        sched_lines = self.get_job_directives()
+        if sched_lines:
+            lines += sched_lines
 
-        if batch_directives_lines:
-            lines.append("####### START OF SCHEDULER DIRECTIVES #######")
-            lines += batch_directives_lines
-            lines.append("####### END OF SCHEDULER DIRECTIVES   #######")
-            lines.append("\n")
+        if self.burstbuffer:
+            burst_buffer_lines = self._get_burst_buffer(self.burstbuffer)
+            if burst_buffer_lines:
+                lines += burst_buffer_lines
+
+        if self.datawarp:
+            data_warp_lines = self._get_data_warp(self.datawarp)
+            if data_warp_lines:
+                lines += data_warp_lines
 
         var_lines = self._get_variables(self.recipe.get("vars"))
         env_lines = self._get_environment(self.recipe.get("env"))
 
-        if self.recipe.get("env"):
+        if deep_get(self.recipe, "executors", self.executor, "env"):
+            env_lines = self._get_environment(
+                self.recipe["executors"][self.executor]["env"]
+            )
+
+        if deep_get(self.recipe, "executors", self.executor, "vars"):
+            var_lines = self._get_variables(
+                self.recipe["executors"][self.executor]["vars"]
+            )
+
+        if env_lines:
             lines += env_lines
 
-        if self.recipe.get("vars"):
+        if var_lines:
             lines += var_lines
 
         if self.recipe.get("pre_cmds"):

buildtest/cli/inspect.py

@@ -113,6 +113,7 @@ def inspect_query(report, args):
                 f"(ID: {test['full_id']})",
                 "{:_<30}".format(""),
             )
+            print("executor: ", test["executor"])
             print("description: ", test["description"])
             print("state: ", test["state"])
             print("returncode: ", test["returncode"])