Removed mock client, improved docs. Fixes #29

docs/usage.md

@@ -11,28 +11,59 @@ from hpcpy import get_client
 client = get_client()
 ```
 
-This will return the most-likely client object based on the submission commands available on the system.
-
-In the case of the factory being unable to return an appropriate client object (or if you need to be explicit), you may import the client explicitly for your system.
-
-For example:
+This will return the most-likely client object based on the submission commands available on the system, raising a `NoClientException` if no host scheduler is detected. In the case of the factory being unable to return an appropriate client object (or if you need to be explicit), you may import the client explicitly for your system:
 
 ```python
-from hpcpy import PBSClient
-client = PBSClient()
+from hpcpy import PBSClient, SLURMClient
+client_pbs = PBSClient()
+client_slurm = SLURMClient()
 ```
 
-You can now use the client object for the remaining examples.
+!!! note
 
-## Submitting jobs
+    When using this approach you are bypassing any auto-detection of the host scheduler.
 
-The simplest way to submit a pre-written job script is via the following command:
+## Submit
 
-```python
-job_id = client.submit("/path/to/script.sh")
-```
+The simplest way to submit a pre-written job script is via the `submit()` command, which executes the appropriate command for the scheduler:
+
+=== "HPCPy (Python)"
+    ```python
+    job_id = client.submit("/path/to/script.sh")
+    ```
+
+=== "PBS"
+    ```shell
+    JOB_ID=$(qsub /path/to/script.sh)
+    ```
+
+=== "SLURM"
+    ```shell
+    JOB_ID=$(sbatch /path/to/script.sh)
+    ```
 
-However, oftentimes it is preferable to use a script template that is rendered with additional variables prior to submission. Depending on how this is written, a single script could be used for multiple scheduling systems.
+### Environment Variables
+
+=== "HPCPy (Python)"
+    ```python
+    job_id = client.submit(
+        "/path/to/script.sh",
+        variables=dict(a=1, b="test")
+    )
+    ```
+
+=== "PBS"
+    ```shell
+    qsub -v a=1,b=test /path/to/script.sh
+    ```
+
+!!! note
+
+    All environment variables are passed to the job as strings WITHOUT treatment of commas.
+
+### Script templates
+
+Script templates can be used to generalise a single template script for use in multiple scenarios (i.e. different scheduling systems).
 
 *template.sh*
 ```shell
@@ -52,6 +83,7 @@ job_id = client.submit(
 ```
 
 This will do two things:
+
 1. The template will be loaded into memory, rendered, and written to a temporary file at `$HOME/.hpcpy/job_scripts` (these are periodically cleared by hpcpy).
 2. The rendered jobscript will be submitted to the scheduler.
 
@@ -68,13 +100,19 @@ job_script_filepath = client._render_job_script(
 )
 ```
 
-## Checking job status
+## Status
 
 Checking the status of a job that has been submitted requires the `job_id` of the job on on the scheduler. Using the `submit()` command as above will return this identifier for use with the client.
 
-```python
-status = client.status(job_id)
-```
+=== "HPCPy (Python)"
+    ```python
+    status = client.status(job_id)
+    ```
+=== "PBS"
+    ```shell
+    STATUS=$(qstat -f -F json $JOB_ID)
+    # ... then grepping through to find the job_state attribute
+    ```
 
 The status will be a character code as listed in `constants.py`, however, certain shortcut methods are available for the most common queries.
 
@@ -88,12 +126,62 @@ client.is_running(job_id)
 
 More shorthand methods will be made available as required.
 
-Note: all status related commands will poll the underlying scheduler; please be mindful of overloading the scheduling system with repeated, frequent calls.
+!!! note
+    All status related commands will poll the underlying scheduler; please be mindful of overloading the scheduling system with repeated, frequent calls.
 
-## Deleting jobs
+## Delete
 
 Deleting a job on the system requires only the `job_id` of the job on the scheduler
 
+=== "HPCPy (Python)"
+    ```python
+    client.delete(job_id)
+    ```
+=== "PBS"
+    ```shell
+    qdel $JOB_ID
+    ```
+
+## Task dependence
+
+HPCpy implements a simple task-dependence strategy at the scheduler level, whereby, we can use scheduler directives to make one job dependent on another.
+
+=== "HPCPy (Python)"
+    ```python
+    job1 = client.submit("job1.sh")
+    job2 = client.submit("job2.sh", depends_on=job1)
+    ```
+=== "PBS"
+    ```shell
+    JOB1=$(qsub job1.sh)
+    JOB2=$(qsub -W depend=afterok:$JOB1 job2.sh)
+    ```
+
+Consider the following snippet:
+
 ```python
-client.delete(job_id)
-```
+from hpcpy import get_client
+client = get_client()
+
+# Submit the first job
+first_id = client.submit("job.sh")
+
+# Submit some interim jobs all requiring the first to finish
+job_ids = list()
+for x in range(3):
+    jobx_id = client.submit("job.sh", depends_on=first_id)
+    job_ids.append(jobx_id)
+
+# Submit a final job that requires everything to have finished.
+job_last = client.submit("job.sh", depends_on=job_ids)
+```
+
+This will create 5 jobs:
+
+- 1 x starting job
+- 3 x middle jobs (which depend on the first)
+- 1 x finishing job (which depends on the middle jobs to complete)
+
+Essentially demonstrating a "fork and join" example.
+
+More advanced graphs can be assembled as needed, the complexity of which is determined by your scheduler.

hpcpy/__init__.py

@@ -4,13 +4,12 @@
 from hpcpy.client.client_factory import ClientFactory
 from hpcpy.client.pbs import PBSClient
 from hpcpy.client.slurm import SlurmClient
-from hpcpy.client.mock import MockClient
 from typing import Union
 
 __version__ = _version.get_versions()["version"]
 
 
-def get_client(*args, **kwargs) -> Union[PBSClient, SlurmClient, MockClient]:
+def get_client(*args, **kwargs) -> Union[PBSClient, SlurmClient]:
     """Get a client object specific for the current scheduler.
 
     Returns

hpcpy/client/client_factory.py

@@ -2,16 +2,14 @@
 
 from hpcpy.client.pbs import PBSClient
 from hpcpy.client.slurm import SlurmClient
-from hpcpy.client.mock import MockClient
-import os
 import hpcpy.exceptions as hx
 from hpcpy.utilities import shell
 from typing import Union
 
 
 class ClientFactory:
 
-    def get_client(*args, **kwargs) -> Union[PBSClient, SlurmClient, MockClient]:
+    def get_client(*args, **kwargs) -> Union[PBSClient, SlurmClient]:
         """Get a client object based on what kind of scheduler we are using.
 
         Arguments:
@@ -21,7 +19,7 @@ def get_client(*args, **kwargs) -> Union[PBSClient, SlurmClient, MockClient]:
 
         Returns
         -------
-        Union[PBSClient, SlurmClient, MockClient]
+        Union[PBSClient, SlurmClient]
             Client object suitable for the detected scheduler.
 
         Raises
@@ -30,11 +28,7 @@ def get_client(*args, **kwargs) -> Union[PBSClient, SlurmClient, MockClient]:
             When no scheduler can be detected.
         """
 
-        clients = dict(ls=MockClient, qsub=PBSClient, sbatch=SlurmClient)
-
-        # Remove the MockClient if dev mode is off
-        if os.getenv("HPCPY_DEV_MODE", "0") != "1":
-            _ = clients.pop("ls")
+        clients = dict(qsub=PBSClient, sbatch=SlurmClient)
 
         # Loop through the clients in order, looking for a valid scheduler
         for cmd, client in clients.items():

hpcpy/client/pbs.py

@@ -27,7 +27,7 @@ def status(self, job_id):
         # Get the status out of the job ID
         _status = parsed.get("Jobs").get(job_id).get("job_state")
         return hc.PBS_STATUSES[_status]
-    
+
     def _render_variables(self, variables):
         """Render the variables flag for PBS.
 
@@ -129,7 +129,7 @@ def submit(
             directives.append(f"-l storage={storage_str}")
             context["storage"] = storage
             context["storage_str"] = storage_str
-        
+
         # Add variables
         if isinstance(variables, dict) and len(variables) > 0:
             directives.append(self._render_variables(variables))

hpcpy/constants.py

@@ -40,11 +40,3 @@
 PBS_SUBMIT = "qsub{directives} {job_script}"
 PBS_STATUS = "qstat -f -F json {job_id}"
 PBS_DELETE = "qdel {job_id}"
-
-# Mock command templateds
-MOCK_SUBMIT = "echo 12345"
-MOCK_STATUS = "echo Q"
-MOCK_DELETE = 'echo "DELETED"'
-
-# Mock status translation
-MOCK_STATUSES = PBS_STATUSES

hpcpy/utilities.py

@@ -8,9 +8,7 @@
 import shlex
 
 
-def shell(
-    cmd, check=True, capture_output=True, **kwargs
-) -> sp.CompletedProcess:
+def shell(cmd, check=True, capture_output=True, **kwargs) -> sp.CompletedProcess:
     """Execute a shell command.
 
     Parameters

mkdocs.yml

@@ -0,0 +1,19 @@
+site_name: HPCPy
+nav:
+  - Home: index.md
+  - Installation: installation.md
+  - Usage: usage.md
+theme: 
+  name: material
+
+markdown_extensions:
+- admonition
+- pymdownx.highlight:
+    anchor_linenums: true
+    line_spans: __span
+    pygments_lang_class: true
+- pymdownx.inlinehilite
+- pymdownx.snippets
+- pymdownx.superfences
+- pymdownx.tabbed:
+    alternate_style: true