Merge pull request #1381 from buildtesters/buildtest_build_limit

add support for limiting builds via 'buildtest build --limit'

bash_completion.sh

@@ -138,7 +138,7 @@ _buildtest ()
   #case "${prev}" in
     build|bd)
       local shortoption="-b -e -et -f -m -s -t -u -x"
-      local longoption="--buildspec --disable-executor-check --executor --executor-type --exclude --filter --helpfilter --maxpendtime --modules --module-purge --nodes --pollinterval --procs --rerun --remove-stagedir --retry --stage --tags --timeout --unload-modules"
+      local longoption="--buildspec --disable-executor-check --executor --executor-type --exclude --filter --helpfilter --limit --maxpendtime --modules --module-purge --nodes --pollinterval --procs --rerun --remove-stagedir --retry --stage --tags --timeout --unload-modules"
       local allopts="${longoption} ${shortoption}"
 
       COMPREPLY=( $( compgen -W "$allopts" -- $cur ) )

buildtest/builders/base.py

@@ -546,7 +546,6 @@ def _write_build_script(self, modules=None, modulepurge=None, unload_modules=Non
         if modules:
             lines.append("# Specify list of modules to load")
             for module in modules.split(","):
-
                 lines.append(f"module load {module}")
 
         lines += [
@@ -716,13 +715,11 @@ def get_job_directives(self):
             lines += [f"#BSUB -e {self.name}.err"]
 
         if self.pbs:
-
             for line in self.pbs:
                 lines.append(f"#PBS {line}")
             lines.append(f"#PBS -N {self.name}")
 
         if self.cobalt:
-
             for line in self.cobalt:
                 lines.append(f"#COBALT {line}")
             lines.append(f"#COBALT --jobname={self.name}")
@@ -868,13 +865,11 @@ def add_metrics(self):
             return
 
         for key in self.metrics.keys():
-
             # default value of metric is empty string
             self.metadata["metrics"][key] = ""
 
             # apply regex on stdout/stderr and assign value to metrics
             if self.metrics[key].get("regex"):
-
                 if self.metrics[key]["regex"]["stream"] == "stdout":
                     content = self._output
                 elif self.metrics[key]["regex"]["stream"] == "stderr":
@@ -962,7 +957,6 @@ def check_test_state(self):
 
         # if status is defined in Buildspec, then check for returncode and regex
         if self.status:
-
             slurm_job_state_match = False
             pbs_job_state_match = False
             lsf_job_state_match = False

buildtest/builders/compiler.py

@@ -429,7 +429,6 @@ def _process_compiler_config(self):
 
         # if default compiler setting provided in buildspec let's assign it.
         if deep_get(self.compiler_section, "default", self.compiler_group):
-
             self.cc = (
                 self.compiler_section["default"][self.compiler_group].get("cc")
                 or self.cc
@@ -462,7 +461,6 @@ def _process_compiler_config(self):
             )
         # if compiler instance defined in config section read from buildspec. This overrides default section if specified
         if deep_get(self.compiler_section, "config", self.compiler):
-
             self.logger.debug(
                 f"[{self.name}]: Detected compiler: {self.compiler} in 'config' scope overriding default compiler group setting for: {self.compiler_group}"
             )

buildtest/builders/script.py

@@ -26,7 +26,6 @@ def __init__(
         numnodes=None,
         compiler=None,
     ):
-
         super().__init__(
             name=name,
             recipe=recipe,
@@ -123,7 +122,6 @@ def generate_script(self):
             lines += sched_lines
 
         if self.burstbuffer:
-
             burst_buffer_lines = self._get_burst_buffer(self.burstbuffer)
             if burst_buffer_lines:
                 lines += burst_buffer_lines

buildtest/builders/spack.py

@@ -211,13 +211,11 @@ def _spack_environment(self, spack_env):
             lines.append(f"spack env rm -y {spack_env['rm']['name']}")
 
         if spack_env.get("create"):
-
             opts = spack_env["create"].get("options") or ""
             cmd = ["spack env create", opts]
 
             # create spack environment from name
             if spack_env["create"].get("name"):
-
                 # if remove_environment is defined we remove the environment before creating it
                 if spack_env["create"].get("remove_environment"):
                     lines.append(f"spack env rm -y {spack_env['create']['name']}")
@@ -246,7 +244,6 @@ def _spack_environment(self, spack_env):
 
             # activate spack environment via directory 'spack env activate -d <dir>'
             elif spack_env["activate"].get("dir"):
-
                 env_dir = resolve_path(spack_env["activate"]["dir"], exist=False)
                 if not env_dir:
                     raise BuildTestError(

buildtest/buildsystem/builders.py

@@ -78,7 +78,6 @@ def __init__(
             return
 
         if deep_get(self.filters, "maintainers"):
-
             if not self.bp.recipe.get("maintainers"):
                 console.print(
                     f"{self.bp.buildspec}: skipping test because [italic]'maintainers'[/italic] field is not specified in buildspec."
@@ -112,7 +111,6 @@ def __init__(
                 # Add the builder for the script or spack schema
 
                 if recipe["type"] in ["script", "compiler", "spack"]:
-
                     builders = self.build(name, recipe)
                     if builders:
                         self.builders += builders
@@ -202,7 +200,6 @@ def create_script_builders(
     def create_compiler_builders(
         self, name, recipe, executor, nodes=None, procs=None, compiler_name=None
     ):
-
         """Create builder objects by calling :class:`buildtest.builders.compiler.CompilerBuilder` class.
 
         args:
@@ -403,7 +400,6 @@ def build(self, name, recipe):
         for compiler_pattern in recipe["compilers"]["name"]:
             for bc_name in discovered_compilers:
                 if re.match(compiler_pattern, bc_name):
-
                     builder = self.generate_builders(
                         name=name, recipe=recipe, compiler_name=bc_name
                     )
@@ -463,7 +459,6 @@ def _skip_tests_by_type(self, recipe, name):
         """
 
         if self.filters.get("type"):
-
             found = self.filters["type"] == recipe["type"]
 
             if not found:

buildtest/buildsystem/checks.py

@@ -35,7 +35,6 @@ def returncode_check(builder):
 
     # if 'returncode' field set for 'status' check the returncode if its not set we return False
     if "returncode" in builder.status.keys():
-
         # returncode can be an integer or list of integers
         buildspec_returncode = builder.status["returncode"]
 

buildtest/buildsystem/parser.py

@@ -175,7 +175,6 @@ def validate(self):
         # with sub schema
 
         for test in self.get_test_names():
-
             self.logger.info(
                 "Validating test - '%s' in recipe: %s" % (test, self.buildspec)
             )

buildtest/cli/__init__.py

@@ -150,7 +150,6 @@ def valid_time(value):
 
 
 def get_parser():
-
     epilog_str = f"""
 References
 
@@ -530,16 +529,6 @@ def build_menu(subparsers):
         type=str,
         help="Specify project account used to charge batch jobs (applicable for batch jobs only)",
     )
-    extra_group.add_argument(
-        "--disable-executor-check",
-        action="store_false",
-        help="Disable executor check during configuration check. By default these checks are enforced for Local, Slurm, PBS, LSF, and Cobalt Executor.",
-    )
-    extra_group.add_argument(
-        "--remove-stagedir",
-        action="store_true",
-        help="Remove stage directory after job completion.",
-    )
     batch_group.add_argument(
         "--maxpendtime",
         type=positive_number,
@@ -562,6 +551,21 @@ def build_menu(subparsers):
         nargs="+",
         type=positive_number,
     )
+    extra_group.add_argument(
+        "--disable-executor-check",
+        action="store_false",
+        help="Disable executor check during configuration check. By default these checks are enforced for Local, Slurm, PBS, LSF, and Cobalt Executor.",
+    )
+    extra_group.add_argument(
+        "--limit",
+        type=positive_number,
+        help="Limit number of tests that can be run.",
+    )
+    extra_group.add_argument(
+        "--remove-stagedir",
+        action="store_true",
+        help="Remove stage directory after job completion.",
+    )
     extra_group.add_argument(
         "--rebuild",
         type=positive_number,

buildtest/cli/build.py

@@ -55,6 +55,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 # Context manager that copies stdout and any exceptions to a log file
 class Tee(object):
     def __init__(self, filename):
@@ -188,7 +189,6 @@ def discover_buildspecs(
     # if user pass buildspecs to be excluded (buildtest build -x <buildspec>) then
     # discover all excluded buildspecs and remove from discovered list
     if exclude_buildspecs:
-
         # discover all excluded buildspecs, if its file add to list,
         # if its directory traverse all .yml files
         for name in exclude_buildspecs:
@@ -242,7 +242,6 @@ def print_discovered_buildspecs(buildspec_dict):
 
     # if any buildspecs removed due to -x option we print them to screen
     if buildspec_dict["excluded"]:
-
         table = Table(
             title="Excluded buildspecs", box=box.DOUBLE_EDGE, header_style="blue"
         )
@@ -254,7 +253,6 @@ def print_discovered_buildspecs(buildspec_dict):
 
     # print breakdown of buildspecs by tags
     if buildspec_dict.get("tags"):
-
         for tagname in buildspec_dict["tags"].keys():
             table = Table(
                 title=f"Buildspecs By Tag={tagname}",
@@ -268,7 +266,6 @@ def print_discovered_buildspecs(buildspec_dict):
 
     # print breakdown of buildspecs by executors
     if buildspec_dict.get("executors"):
-
         for executorname in buildspec_dict["executors"].keys():
             table = Table(
                 title=f"Buildspecs by Executor={executorname}",
@@ -319,7 +316,6 @@ def discover_buildspecs_by_tags(buildspec_cache, tagnames):
 
         for buildspecfile in buildspec_cache["buildspecs"].keys():
             for test in buildspec_cache["buildspecs"][buildspecfile].keys():
-
                 # if input tag is not of type str we skip the tag name since it is not valid
                 if not isinstance(name, str):
                     logger.warning(f"Tag: {name} is not of type 'str'")
@@ -366,7 +362,6 @@ def discover_buildspecs_by_executor(buildspec_cache, executors):
 
         for buildspecfile in buildspec_cache["buildspecs"].keys():
             for test in buildspec_cache["buildspecs"][buildspecfile].keys():
-
                 # check if executor in buildspec matches one in argument (buildtest build --executor <EXECUTOR>)
                 if name == buildspec_cache["buildspecs"][buildspecfile][test].get(
                     "executor"
@@ -500,6 +495,7 @@ def __init__(
         rerun=None,
         executor_type=None,
         timeout=None,
+        limit=None,
     ):
         """The initializer method is responsible for checking input arguments for type
         check, if any argument fails type check we raise an error. If all arguments pass
@@ -531,6 +527,7 @@ def __init__(
             rerun (bool, optional): Rerun last successful **buildtest build** command. This is specified via ``buildtest build --rerun``. All other options will be ignored and buildtest will read buildtest options from file **BUILDTEST_RERUN_FILE**.
             executor_type (bool, optional): Filter test by executor type. This option will filter test after discovery by local or batch executors. This can be specified via ``buildtest build --exec-type``
             timeout (int, optional): Test timeout in seconds specified by ``buildtest build --timeout``
+            limit (int, optional): Limit number of tests that can be run. This option is specified by ``buildtest build --limit``
         """
 
         if buildspecs and not isinstance(buildspecs, list):
@@ -568,6 +565,12 @@ def __init__(
             if timeout <= 0:
                 raise BuildTestError("Timeout must be greater than 0")
 
+        if limit is not None:
+            if not isinstance(limit, int):
+                raise BuildTestError(f"{timeout} is not of type int")
+            if limit <= 0:
+                raise BuildTestError("Limit must be greater than 0")
+
         self.remove_stagedir = remove_stagedir
         self.configuration = configuration
         self.buildspecs = buildspecs
@@ -590,6 +593,7 @@ def __init__(
         self.numnodes = numnodes
         self.executor_type = executor_type
         self.timeout = timeout
+        self.limit = limit
 
         # this variable contains the detected buildspecs that will be processed by buildtest.
         self.detected_buildspecs = None
@@ -614,7 +618,6 @@ def __init__(
         )
 
         if self.logdir:
-
             create_dir(self.logdir)
             self.logfile.name = os.path.join(
                 self.logdir, os.path.basename(self.logfile.name)
@@ -677,7 +680,8 @@ def __init__(
     def load_rerun_file(self):
         """This will load content of file BUILDTEST_RERUN_FILE that contains a dictionary of key/value pair
         that keeps track of last ``buildtest build`` command. This is used with ``buildtest build --rerun``. Upon loading
-        file we reinitalize all class variables that store argument for ``buildtest build`` options"""
+        file we reinitalize all class variables that store argument for ``buildtest build`` options
+        """
 
         if not is_file(BUILDTEST_RERUN_FILE):
             raise BuildTestError(
@@ -715,6 +719,7 @@ def load_rerun_file(self):
         self.numprocs = content["numprocs"]
         self.executor_type = content["executor_type"]
         self.timeout = content["timeout"]
+        self.limit = content["limit"]
 
     def save_rerun_file(self):
         """Record buildtest command options and save them into rerun file which is read when invoking ``buildtest build --rerun``."""
@@ -741,6 +746,7 @@ def save_rerun_file(self):
             "numnodes": self.numnodes,
             "executor_type": self.executor_type,
             "timeout": self.timeout,
+            "limit": self.limit,
         }
 
         with open(BUILDTEST_RERUN_FILE, "w") as fd:
@@ -806,6 +812,12 @@ def build(self):
         if not self.builders or self.stage == "parse":
             return
 
+        if self.limit:
+            self.builders = self.builders[: self.limit]
+            console.print(
+                f"[red]Limit number of tests to {self.limit} for Building and Running. "
+            )
+
         self.build_phase()
 
         # if --stage=build is set  we return from method
@@ -894,14 +906,12 @@ def parse_buildspecs(self):
         console.print(f"[red]Invalid Buildspecs: {len(self.invalid_buildspecs)}")
 
         for buildspec in valid_buildspecs:
-
             msg = f"[green]{buildspec}: VALID"
             console.print(msg)
 
         # print any skipped buildspecs if they failed to validate during build stage
         if self.invalid_buildspecs:
             for buildspec in self.invalid_buildspecs:
-
                 msg = f"[red]{buildspec}: INVALID"
                 console.print(msg)
 
@@ -1322,7 +1332,6 @@ def print_batch_builders(self, builders):
     def print_builders(
         self, compiler_builder, spack_builder, script_builder, batch_builder
     ):
-
         """Print detected builders during build phase"""
 
         self.print_builders_by_type(script_builder, builder_type="script")