More batch keys

buildtest/buildsystem/batch.py

@@ -5,13 +5,20 @@ def get_headers(self):
 
 class LSFBatchScript(BatchScript):
     batch_translation = {
-        "queue": "-q",
-        "nodecount": "-nnodes",
-        "cpucount": "-n",
-        "timelimit": "-W",
-        "memory": "-M",
         "account": "-P",
+        "begintime": "-b",
+        "cpucount": "-n",
+        "email-address": "-u",
         "exclusive": "-x",
+        "memory": "-M",
+        "network": "-network",
+        "nodecount": "-nnodes",
+        "qos": None,
+        "queue": "-q",
+        "tasks-per-core": None,
+        "tasks-per-node": None,
+        "tasks-per-socket": None,
+        "timelimit": "-W",
     }
 
     def __init__(self, batch=None, bsub=None):
@@ -45,6 +52,9 @@ def build_header(self):
                     self.headers += [
                         f"{self.directive} {self.batch_translation['exclusive']}"
                     ]
+                # if batch key is None that means scheduler doesn't support the option
+                elif not self.batch_translation.get(key):
+                    continue
                 else:
                     self.headers += [
                         f"{self.directive} {self.batch_translation[key]} {value}"
@@ -53,13 +63,20 @@ def build_header(self):
 
 class SlurmBatchScript(BatchScript):
     batch_translation = {
-        "queue": "--partition",
-        "nodecount": "--nodes",
-        "cpucount": "--ntasks",
-        "timelimit": "--time",
-        "memory": "--mem",
         "account": "--account",
+        "begintime": "--begin",
+        "cpucount": "--ntasks",
+        "email-address": "--mail-user",
         "exclusive": "--exclusive",
+        "memory": "--mem",
+        "network": "--network",
+        "nodecount": "--nodes",
+        "qos": "--qos",
+        "queue": "--partition",
+        "tasks-per-core": "--ntasks-per-core",
+        "tasks-per-node": "--ntasks-per-node",
+        "tasks-per-socket": "--ntasks-per-socket",
+        "timelimit": "--time",
     }
 
     def __init__(self, batch=None, sbatch=None):
@@ -90,6 +107,9 @@ def build_header(self):
                     self.headers += [
                         f"{self.directive} {self.batch_translation['exclusive']}=user"
                     ]
+                # if batch key is None that means scheduler doesn't support the option
+                elif not self.batch_translation.get(key):
+                    continue
                 else:
                     self.headers += [
                         f"{self.directive} {self.batch_translation[key]}={value}"

buildtest/schemas/definitions.schema.json

@@ -75,34 +75,63 @@
     "batch": {
       "type": "object",
       "description": "The ``batch`` field is used to specify scheduler agnostic directives that are translated to #SBATCH or #BSUB based on your scheduler. This is an experimental feature that supports a subset of scheduler parameters.",
+      "additionalProperties": false,
       "properties": {
-        "queue": {
-          "type":  "string",
-          "description": "Specify Job Queue"
-          },
-        "nodecount": {
+        "account": {
           "type": "string",
-          "description": "Specify number of Nodes to allocate"
+          "description": "Specify Account to charge job"
+        },
+        "begintime": {
+          "type": "string",
+          "description": "Specify begin time when job will start allocation"
         },
         "cpucount": {
           "type": "string",
           "description": "Specify number of CPU to allocate"
         },
-        "timelimit": {
+        "email-address": {
           "type": "string",
-          "description": "Specify Job timelimit"
+          "description": "Email Address to notify on Job State Changes"
+        },
+        "exclusive": {
+          "type": "boolean",
+          "description": "Specify if job needs to run in exclusive mode"
         },
         "memory": {
           "type": "string",
           "description": "Specify Memory Size for Job"
         },
-        "account": {
+        "network": {
           "type": "string",
-          "description": "Specify Account to charge job"
+          "description": "Specify network resource requirement for job"
         },
-        "exclusive": {
-          "type": "boolean",
-          "description": "Specify if job needs to run in exclusive mode"
+        "nodecount": {
+          "type": "string",
+          "description": "Specify number of Nodes to allocate"
+        },
+        "qos": {
+          "type": "string",
+          "description": "Specify Quality of Service (QOS)"
+        },
+        "queue": {
+          "type":  "string",
+          "description": "Specify Job Queue"
+          },
+        "tasks-per-core": {
+          "type": "string",
+          "description": "Request number of tasks to be invoked on each core. "
+        },
+        "tasks-per-node": {
+          "type": "string",
+          "description": "Request number of tasks to be invoked on each node. "
+        },
+        "tasks-per-socket": {
+          "type": "string",
+          "description": "Request the maximum tasks to be invoked on each socket. "
+        },
+        "timelimit": {
+          "type": "string",
+          "description": "Specify Job timelimit"
         }
       }
     },

buildtest/schemas/examples/compiler-v1.0.schema.json/invalid/examples.yml

@@ -123,6 +123,21 @@ buildspecs:
       cflags: "-O1"
       ldflags: "-lm"
 
+  additionalProperties_batch:
+    executor: local.bash
+    type: compiler
+    description: "test additionalProperties in batch field"
+    module:
+    - "module purge && module load gcc/4.0"
+    batch:
+      "nodecount": "2"
+      "EXPORT": "ALL"
+    build:
+      source: src/hello.c
+      name: gnu
+      cflags: "-O1"
+      ldflags: "-lm"
+
   type_mismatch_args:
     executor: local.bash
     type: compiler

buildtest/schemas/examples/compiler-v1.0.schema.json/valid/examples.yml

@@ -101,6 +101,53 @@ buildspecs:
       name: gnu
       cxx: g++
 
+  batch_example:
+    type: compiler
+    description: example using batch field
+    executor: lsf.batch
+    batch:
+      "timelimit": "30"
+      "nodecount": "2"
+      "queue": "batch"
+      "account": "biology"
+    build:
+      source: "src/hello.cpp"
+      name: gnu
+      cxx: g++
+
+
+  batch_bsub_example:
+    type: compiler
+    description: example using batch with bsub key
+    executor: lsf.batch
+    batch:
+      "timelimit": "30"
+      "nodecount": "2"
+      "queue": "batch"
+      "account": "biology"
+    bsub: ["-n 4"]
+    build:
+      source: "src/hello.cpp"
+      name: gnu
+      cxx: g++
+
+
+  batch_sbatch_example:
+    type: compiler
+    description: example using batch with sbatch key
+    executor: lsf.batch
+    batch:
+      "timelimit": "30"
+      "nodecount": "2"
+      "queue": "batch"
+      "account": "biology"
+    sbatch: ["--ntasks=4"]
+    build:
+      source: "src/hello.cpp"
+      name: gnu
+      cxx: g++
+
+
   args_example:
     type: compiler
     description: Launcher example

docs/buildspecs/batch_support.rst

@@ -280,17 +280,25 @@ script directive supported by the scheduler. Shown below is a translation table
 for the **batch** field
 
 
-.. csv-table::
+.. csv-table:: Batch Translation Table
    :header: "Field", "Slurm", "LSF"
    :widths: 25 25 25
 
-   **queue**, --partition, -q
-   **nodecount**, --nodes, -nnodes
+   **account**, --account, -P
+   **begin**, --begin, -b
    **cpucount**, --ntasks, -n
-   **timelimit**, --time, -W
+   **email-address**, --mail-user, -u
+   **exclusive**, --exclusive=user, -x
    **memory**, --mem, -M
-   **account**, --account, -P
-   **exclusive**, --nodes, -x
+   **network**, --network, -network
+   **nodecount**, --nodes, -nnodes
+   **qos**, --qos, N/A
+   **queue**, --partition, -q
+   **tasks-per-core**, --ntasks-per-core, N/A
+   **tasks-per-node**, --ntasks-per-node, N/A
+   **tasks-per-socket**, --ntasks-per-socket, N/A
+   **timelimit**, --time, -W
+
 
 In this example, we rewrite the LSF buildspec to use ``batch`` instead of ``bsub``
 field::

tests/buildsystem/test_batch.py

@@ -0,0 +1,58 @@
+from buildtest.buildsystem.batch import LSFBatchScript, SlurmBatchScript
+
+
+def test_batchscript_example1():
+    batch_cmds = {
+        "account": "biology",
+        "cpucount": "1",
+        "memory": "500MB",
+        "queue": "debug",
+        "timelimit": "10",
+    }
+    expected_header = [
+        "#BSUB -P biology",
+        "#BSUB -n 1",
+        "#BSUB -M 500MB",
+        "#BSUB -q debug",
+        "#BSUB -W 10",
+    ]
+    script = LSFBatchScript(batch_cmds)
+    header = script.get_headers()
+    assert expected_header == header
+    print("actual header %s and expected header: %s" % (header, expected_header))
+
+    script = SlurmBatchScript(batch_cmds)
+    expected_header = [
+        "#SBATCH --account=biology",
+        "#SBATCH --ntasks=1",
+        "#SBATCH --mem=500MB",
+        "#SBATCH --partition=debug",
+        "#SBATCH --time=10",
+    ]
+    header = script.get_headers()
+    assert expected_header == header
+    print("actual header %s and expected header: %s" % (header, expected_header))
+
+
+def test_batchscript_example2():
+    batch_cmds = {
+        "tasks-per-node": "1",
+        "cpucount": "2",
+    }
+    sbatch_cmd = ["-t 10", "-q normal"]
+    script = SlurmBatchScript(batch_cmds, sbatch_cmd)
+    header = script.get_headers()
+    expected_header = [
+        "#SBATCH -t 10",
+        "#SBATCH -q normal",
+        "#SBATCH --ntasks-per-node=1",
+        "#SBATCH --ntasks=2",
+    ]
+    assert expected_header == header
+
+    bsub_cmds = ["-W 10", "-q normal"]
+    script = LSFBatchScript(batch_cmds, bsub_cmds)
+    header = script.get_headers()
+    # tasks-per-node is not valid option in LSF so this option is skipped. bsub commands are processed first before batch
+    expected_header = ["#BSUB -W 10", "#BSUB -q normal", "#BSUB -n 2"]
+    assert expected_header == header