Merge pull request #831 from jmchilton/toil

Add ``--toil`` engine type for CWL development.

planemo/cwl/__init__.py

@@ -1,9 +1,11 @@
 """Entry point for modules describing abstractions for dealing with CWL artifacts."""
 from .run import run_cwltool
 from .script import to_script
+from .toil import run_toil
 
 
 __all__ = (
     'run_cwltool',
+    'run_toil',
     'to_script',
 )

planemo/cwl/toil.py

@@ -0,0 +1,80 @@
+from __future__ import absolute_import
+
+import json
+import tempfile
+
+try:
+    from toil.cwl import cwltoil
+except ImportError:
+    cwltoil = None
+
+from planemo.deps import ensure_dependency_resolvers_conf_configured
+from planemo.io import error, real_io
+from planemo.runnable import (
+    ErrorRunResponse,
+)
+from .run import (
+    CwlToolRunResponse,
+    JSON_PARSE_ERROR_MESSAGE
+)
+
+
+TOIL_REQUIRED_MESSAGE = "This functionality requires Toil, please install with 'pip install toil'"
+
+
+def run_toil(ctx, path, job_path, **kwds):
+    """Translate planemo kwds to cwltool kwds and run cwltool main function."""
+    _ensure_toil_available()
+
+    args = []
+    if not ctx.verbose:
+        args.append("--quiet")
+    output_directory = kwds.get("output_directory", None)
+    if output_directory:
+        args.append("--outdir")
+        args.append(output_directory)
+    if kwds.get("no_container", False):
+        args.append("--no-container")
+        ensure_dependency_resolvers_conf_configured(ctx, kwds)
+        args.append("--beta-dependency-resolvers-configuration")
+        args.append(kwds["dependency_resolvers_config_file"])
+    if kwds.get("mulled_containers"):
+        args.append("--beta-use-biocontainers")
+
+    if kwds.get("non_strict_cwl", False):
+        args.append("--non-strict")
+
+    args.extend([path, job_path])
+    ctx.vlog("Calling cwltoil with arguments %s" % args)
+    with tempfile.NamedTemporaryFile("w") as tmp_stdout:
+        # cwltool passes sys.stderr to subprocess.Popen - ensure it has
+        # and actual fileno.
+        with real_io():
+            ret_code = cwltoil.main(
+                args,
+                stdout=tmp_stdout
+            )
+        tmp_stdout.flush()
+        with open(tmp_stdout.name, "r") as stdout_f:
+            try:
+                result = json.load(stdout_f)
+            except ValueError:
+                message = JSON_PARSE_ERROR_MESSAGE % (
+                    open(tmp_stdout.name, "r").read(),
+                    tmp_stdout.name,
+                )
+                error(message)
+                raise Exception(message)
+
+        if ret_code != 0:
+            return ErrorRunResponse("Error running Toil")
+        outputs = result
+    return CwlToolRunResponse(
+        "",
+        outputs=outputs,
+    )
+
+
+def _ensure_toil_available():
+    if cwltoil is None:
+        raise Exception(TOIL_REQUIRED_MESSAGE)

planemo/engine/factory.py

@@ -8,6 +8,7 @@
     ExternalGalaxyEngine,
     LocalManagedGalaxyEngine,
 )
+from .toil import ToilEngine
 
 
 UNKNOWN_ENGINE_TYPE_MESSAGE = "Unknown engine type specified [%s]."
@@ -30,6 +31,8 @@ def build_engine(ctx, **kwds):
         engine_type = ExternalGalaxyEngine
     elif engine_type_str == "cwltool":
         engine_type = CwlToolEngine
+    elif engine_type_str == "toil":
+        engine_type = ToilEngine
     else:
         raise Exception(UNKNOWN_ENGINE_TYPE_MESSAGE % engine_type_str)
 

planemo/engine/toil.py

@@ -0,0 +1,24 @@
+"""Module contianing the :class:`ToilEngine` implementation of :class:`Engine`."""
+
+from planemo import cwl
+from planemo.runnable import RunnableType
+from .interface import BaseEngine
+
+
+class ToilEngine(BaseEngine):
+    """An :class:`Engine` implementation backed by Toil.
+
+    More information on toil can be found at https://github.com/BD2KGenomics/toil.
+    """
+
+    handled_runnable_types = [RunnableType.cwl_tool, RunnableType.cwl_workflow]
+
+    def _run(self, runnable, job_path):
+        """Run CWL job using Toil."""
+        path = runnable.path
+        return cwl.run_toil(self._ctx, path, job_path, **self._kwds)
+
+
+__all__ = (
+    "ToilEngine",
+)

planemo/options.py

@@ -33,7 +33,7 @@ def skip_venv_option():
 def run_engine_option():
     return planemo_option(
         "--engine",
-        type=click.Choice(["galaxy", "docker_galaxy", "cwltool", "external_galaxy"]),
+        type=click.Choice(["galaxy", "docker_galaxy", "cwltool", "toil", "external_galaxy"]),
         default="galaxy",
         use_global_config=True,
         help=("Select an engine to run or test aritfacts such as tools "

planemo/runnable.py

@@ -264,6 +264,9 @@ def _check_output(self, output_id, output_value, output_test):
             if not isinstance(output_value, dict):
                 output_problems.append("Expected file properties for output [%s]" % output_id)
                 return
+            if "path" not in output_value and "location" in output_value:
+                assert output_value["location"].startswith("file://")
+                output_value["path"] = output_value["location"][len("file://"):]
             if "path" not in output_value:
                 output_problems.append("No path specified for expected output file [%s]" % output_id)
                 return