Merge pull request #35 from amq92/slurm-test

Add slurm github action

Author: amq92

.github/workflows/python-run-tests.yml

@@ -10,15 +10,39 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        ports:
+          - "8888:3306"
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install build
-    - name: Run tests with unittest
-      run: python -m unittest -v 
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up SLURM
+        uses: koesterlab/setup-slurm-action@v1
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+
+      - name: Copy files to SLURM node
+        run: |
+          mkdir -p $HOME/slurm_workspace
+          rsync -av --progress . $HOME/slurm_workspace/
+          ls -lah $HOME/slurm_workspace  # Debugging step
+
+      - name: Run tests with unittest
+        run: |
+          cd $HOME/slurm_workspace
+          python -m unittest -v

CONTRIBUTING.md

@@ -20,16 +20,17 @@ We welcome contributions! Before submitting changes, please follow these guideli
    ```
 4. Test/validate changes (see [Testing](#testing)).
 5. Update documentation if needed.
-6. Open a PR with a clear description of changes and test results.
+6. Open a Pull Request with a clear description of changes and test results.
 
 ## Code Style
 - Format code with [`ruff`](https://docs.astral.sh/ruff/).
 - Add type hints for new functions/methods.
 - Keep docstrings consistent with existing code.
 
 ## Testing
-- Testing on a real Slurm cluster is **highly desired**.
 - Add unit tests to validate any change in functionality.
+- Testing on a real Slurm cluster is **highly desired**.
+- A simple Slurm cluster is setup as an automatic action for any Pull Request.
 
 ## Questions?
 - Open a GitHub issue.

test/test_cli.py

@@ -55,52 +55,19 @@ def test_03_using_equal_and_spaces(self):
         )
 
     def __run_test(self, args):
-        stdout = run_cli(args)
-        script, job_id = parse_stdout(stdout)
+        with io.StringIO() as buffer:
+            with contextlib.redirect_stdout(buffer):
+                with patch.object(sys, "argv", args):
+                    cli()
+                    stdout = buffer.getvalue()
 
-        out_file = f"slurm-{job_id}.out"
-        while True:  # wait for job to finalize
-            if os.path.isfile(out_file):
-                break
-        with open(out_file, "r") as fid:
-            contents = fid.read()
-        os.remove(out_file)
+        *script, _, job_msg = stdout.strip().split("\n")
+        script = "\n".join(script).strip()
+        job_id = int(job_msg.replace("Submitted batch job ", ""))
 
         self.assertEqual(self.script, script)
-        self.assertIn("Hello!", contents)
         self.assertIn(f"Submitted batch job {job_id}", stdout)
 
 
-def run_cli(testargs):
-    with io.StringIO() as buffer:
-        with contextlib.redirect_stdout(buffer):
-            with patch.object(sys, "argv", testargs):
-                if shutil.which("sbatch") is not None:
-                    cli()
-                else:
-                    with patch("subprocess.run", subprocess_sbatch):
-                        cli()
-                stdout = buffer.getvalue()
-                return stdout
-
-
-def subprocess_sbatch(*args, **kwargs):
-    job_id = 1234
-    out_file = f"slurm-{job_id}.out"
-    with open(out_file, "w") as fid:
-        fid.write("Hello!!!\n")
-    stdout = f"Submitted batch job {job_id}"
-    return subprocess.CompletedProcess(
-        *args, returncode=1, stdout=stdout.encode("utf-8")
-    )
-
-
-def parse_stdout(stdout):
-    *script, _, job_msg = stdout.strip().split("\n")
-    script = "\n".join(script).strip()
-    job_id = int(job_msg.replace("Submitted batch job ", ""))
-    return script, job_id
-
-
 if __name__ == "__main__":
     unittest.main()

test/test_core.py

@@ -5,7 +5,6 @@
 import shutil
 import subprocess
 import unittest
-from unittest.mock import patch
 
 from simple_slurm import Slurm
 
@@ -209,34 +208,15 @@ def test_13_output_env_vars(self):
 
     def test_14_srun_returncode(self):
         slurm = Slurm(contiguous=True)
-        if shutil.which("srun") is not None:
-            code = slurm.srun("echo Hello!")
-        else:
-            with patch("subprocess.run", subprocess_srun):
-                code = slurm.srun("echo Hello!")
+        code = slurm.srun("echo Hello!")
         self.assertEqual(code, 0)
 
     def test_15_sbatch_execution(self):
-        with io.StringIO() as buffer:
-            with contextlib.redirect_stdout(buffer):
-                slurm = Slurm(contiguous=True)
-                if shutil.which("sbatch") is not None:
-                    job_id = slurm.sbatch("echo Hello!")
-                else:
-                    with patch("subprocess.run", subprocess_sbatch):
-                        job_id = slurm.sbatch("echo Hello!")
-                stdout = buffer.getvalue()
+        slurm = Slurm(contiguous=True)
+        job_id, stdout = self.__run_sbatch(slurm)
 
-        out_file = f"slurm-{job_id}.out"
-        while True:  # wait for job to finalize
-            if os.path.isfile(out_file):
-                break
-        with open(out_file, "r") as fid:
-            contents = fid.read()
-        os.remove(out_file)
         self.assertFalse(slurm.is_parsable)
         self.assertIsInstance(job_id, int)
-        self.assertIn("Hello!", contents)
         self.assertIn(f"Submitted batch job {job_id}", stdout)
 
     def test_16_parse_timedelta(self):
@@ -281,36 +261,19 @@ def test_18_false_boolean_arguments(self):
 
     def test_19_sbatch_execution_with_job_file(self):
         job_file = "script.sh"
-        with io.StringIO() as buffer:
-            with contextlib.redirect_stdout(buffer):
-                slurm = Slurm(contiguous=True)
-                if shutil.which("sbatch") is not None:
-                    job_id = slurm.sbatch("echo Hello!", job_file=job_file)
-                else:
-                    with patch("subprocess.run", subprocess_sbatch):
-                        job_id = slurm.sbatch("echo Hello!", job_file=job_file)
-                stdout = buffer.getvalue()
 
+        slurm = Slurm(contiguous=True)
+        job_id, stdout = self.__run_sbatch(slurm, job_file=job_file)
+
+        self.assertFalse(slurm.is_parsable)
         self.assertIsInstance(job_id, int)
+        self.assertIn(f"Submitted batch job {job_id}", stdout)
 
-        out_file = f"slurm-{job_id}.out"
-        while True:  # wait for job to finalize
-            if os.path.isfile(out_file):
-                break
-        # Assert the script was written correctly
         with open(job_file, "r") as fid:
             job_contents = fid.read()
         os.remove(job_file)
 
-        self.assertEqual(job_contents, self.job_file_test_19)
-
-        # Assert the script was executed correctly
-        with open(out_file, "r") as fid:
-            contents = fid.read()
-        os.remove(out_file)
-
-        self.assertIn("Hello!", contents)
-        self.assertIn(f"Submitted batch job {job_id}", stdout)
+        self.assertEqual(self.job_file_test_19, job_contents)
 
     def test_20_add_cmd_single(self):
         slurm = Slurm(
@@ -367,54 +330,19 @@ def test_21_add_cmd_multiple(self):
         self.assertEqual(self.script + "\n" + self.commands, str(slurm))
 
     def test_22_parsable_sbatch_execution(self):
-        with io.StringIO() as buffer:
-            with contextlib.redirect_stdout(buffer):
-                slurm = Slurm(contiguous=True, parsable=True)
-                if shutil.which("sbatch") is not None:
-                    job_id = slurm.sbatch("echo Hello!")
-                else:
-                    with patch("subprocess.run", subprocess_sbatch_parsable):
-                        job_id = slurm.sbatch("echo Hello!")
-                stdout = buffer.getvalue()
+        slurm = Slurm(contiguous=True, parsable=True)
+        job_id, stdout = self.__run_sbatch(slurm)
 
-        out_file = f"slurm-{job_id}.out"
-        while True:  # wait for job to finalize
-            if os.path.isfile(out_file):
-                break
-        with open(out_file, "r") as fid:
-            contents = fid.read()
-        os.remove(out_file)
         self.assertTrue(slurm.is_parsable)
         self.assertIsInstance(job_id, int)
-        self.assertIn("Hello!", contents)
         self.assertEqual(f"{job_id}\n", stdout)
 
-
-def subprocess_srun(*args, **kwargs):
-    print("Hello!!!")
-    return subprocess.CompletedProcess(*args, returncode=0)
-
-
-def subprocess_sbatch(*args, **kwargs):
-    job_id = 1234
-    out_file = f"slurm-{job_id}.out"
-    with open(out_file, "w") as fid:
-        fid.write("Hello!!!\n")
-    stdout = f"Submitted batch job {job_id}"
-    return subprocess.CompletedProcess(
-        *args, returncode=1, stdout=stdout.encode("utf-8")
-    )
-
-
-def subprocess_sbatch_parsable(*args, **kwargs):
-    job_id = 1234
-    out_file = f"slurm-{job_id}.out"
-    with open(out_file, "w") as fid:
-        fid.write("Hello!!!\n")
-    stdout = str(job_id)
-    return subprocess.CompletedProcess(
-        *args, returncode=0, stdout=stdout.encode("utf-8")
-    )
+    def __run_sbatch(self, slurm, *args, **kwargs):
+        with io.StringIO() as buffer:
+            with contextlib.redirect_stdout(buffer):
+                job_id = slurm.sbatch("echo Hello!", *args, **kwargs)
+                stdout = buffer.getvalue()
+        return job_id, stdout
 
 
 if __name__ == "__main__":