[WIP] cleanups

cwl-runner/setup.py

@@ -1,11 +1,7 @@
 #!/usr/bin/env python
 
 import os
-import sys
-import setuptools.command.egg_info as egg_info_cmd
-import shutil
-
-from setuptools import setup, find_packages
+from setuptools import setup
 
 SETUP_DIR = os.path.dirname(__file__)
 README = os.path.join(SETUP_DIR, 'README')
@@ -16,14 +12,16 @@
       long_description=open(README).read(),
       author='Common workflow language working group',
       author_email='common-workflow-language@googlegroups.com',
-      url="https://github.com/common-workflow-language/common-workflow-language",
-      download_url="https://github.com/common-workflow-language/common-workflow-language",
+      url="https://github.com/common-workflow-language/"
+      "common-workflow-language",
+      download_url="https://github.com/common-workflow-language/"
+      "common-workflow-language",
       license='Apache 2.0',
       install_requires=[
           'cwltool'
-        ],
+      ],
       entry_points={
-          'console_scripts': [ "cwl-runner=cwltool.main:main" ]
+          'console_scripts': ["cwl-runner=cwltool.main:main"]
       },
       zip_safe=True
-)
+      )

cwltool/__main__.py

@@ -1,4 +1,4 @@
-import main
+from . import main
 import sys
 
 sys.exit(main.main())

cwltool/aslist.py

@@ -1,4 +1,7 @@
-def aslist(l):
+from typing import Any
+
+
+def aslist(l):  # type: (Any) -> List[Any]
     if isinstance(l, list):
         return l
     else:

cwltool/builder.py

@@ -1,27 +1,58 @@
 import copy
-from aslist import aslist
-import expression
+from .aslist import aslist
+from . import expression
 import avro
 import schema_salad.validate as validate
+from typing import Any, Union
+from .errors import WorkflowException
+from .stdfsaccess import StdFsAccess
 
 CONTENT_LIMIT = 64 * 1024
 
-def substitute(value, replace):
+
+def substitute(value, replace):  # type: (str, str) -> str
     if replace[0] == "^":
         return substitute(value[0:value.rindex('.')], replace[1:])
     else:
         return value + replace
 
+
+def tostr(value):  # type: (Union[dict[str,str],Any]) -> str
+    if isinstance(value, dict) and value.get("class") == "File":
+        if "path" not in value:
+            raise WorkflowException(
+                "File object must have \"path\": %s" % (value))
+        return value["path"]
+    else:
+        return str(value)
+
+
 class Builder(object):
 
+    def __init__(self):  # type: () -> None
+        self.names = None  # type: avro.schema.Names
+        self.schemaDefs = None  # type: Dict[str,Dict[str,str]]
+        self.files = None  # type: List[str]
+        self.fs_access = None  # type: StdFsAccess
+        self.job = None  # type: Dict[str,str]
+        self.requirements = None  # type: Dict[str,str]
+        self.outdir = None  # type: str
+        self.tmpdir = None  # type: str
+        self.resources = None  # type: Dict[str,str]
+        self.bindings = None  # type: List[Dict[str,str]]
+        self.timeout = None  # type: int
+
     def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
-        bindings = []
-        binding = None
-        if "inputBinding" in schema and isinstance(schema["inputBinding"], dict):
+        # type: (Dict[str,Any], Any, List[int], List[int]) -> List[Dict[str,str]]
+        bindings = []  # type: List[Dict[str,str]]
+        binding = None  # type: Dict[str,Any]
+        if "inputBinding" in schema and isinstance(
+                schema["inputBinding"], dict):
             binding = copy.copy(schema["inputBinding"])
 
             if "position" in binding:
-                binding["position"] = aslist(lead_pos) + aslist(binding["position"]) + aslist(tail_pos)
+                binding["position"] = aslist(lead_pos) + aslist(
+                    binding["position"]) + aslist(tail_pos)
             else:
                 binding["position"] = aslist(lead_pos) + [0] + aslist(tail_pos)
 
@@ -34,28 +65,39 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
             for t in schema["type"]:
                 if isinstance(t, basestring) and self.names.has_name(t, ""):
                     avsc = self.names.get_name(t, "")
-                elif isinstance(t, dict) and "name" in t and self.names.has_name(t["name"], ""):
+                elif (isinstance(t, dict) and
+                      "name" in t and self.names.has_name(t["name"], "")):
                     avsc = self.names.get_name(t["name"], "")
                 else:
                     avsc = avro.schema.make_avsc_object(t, self.names)
                 if validate.validate(avsc, datum):
                     schema = copy.deepcopy(schema)
                     schema["type"] = t
-                    return self.bind_input(schema, datum, lead_pos=lead_pos, tail_pos=tail_pos)
-            raise validate.ValidationException("'%s' is not a valid union %s" % (datum, schema["type"]))
+                    return self.bind_input(schema, datum, lead_pos=lead_pos,
+                                           tail_pos=tail_pos)
+            raise validate.ValidationException(
+                "'%s' is not a valid union %s" % (datum, schema["type"]))
         elif isinstance(schema["type"], dict):
             st = copy.deepcopy(schema["type"])
-            if binding and "inputBinding" not in st and "itemSeparator" not in binding and st["type"] in ("array", "map"):
+            if (binding and
+                    "inputBinding" not in st and
+                    "itemSeparator" not in binding and
+                    st["type"] in ("array", "map")):
                 st["inputBinding"] = {}
-            bindings.extend(self.bind_input(st, datum, lead_pos=lead_pos, tail_pos=tail_pos))
+            bindings.extend(
+                self.bind_input(st, datum, lead_pos=lead_pos,
+                                tail_pos=tail_pos))
         else:
             if schema["type"] in self.schemaDefs:
                 schema = self.schemaDefs[schema["type"]]
 
             if schema["type"] == "record":
                 for f in schema["fields"]:
                     if f["name"] in datum:
-                        bindings.extend(self.bind_input(f, datum[f["name"]], lead_pos=lead_pos, tail_pos=f["name"]))
+                        bindings.extend(
+                            self.bind_input(
+                                f, datum[f["name"]], lead_pos=lead_pos,
+                                tail_pos=f["name"]))
                     else:
                         datum[f["name"]] = f.get("default")
 
@@ -65,8 +107,10 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
                     if binding:
                         b2 = copy.deepcopy(binding)
                         b2["valueFrom"] = [n, item]
-                    bindings.extend(self.bind_input({"type": schema["values"], "inputBinding": b2},
-                                                    item, lead_pos=n, tail_pos=tail_pos))
+                    bindings.extend(
+                        self.bind_input(
+                            {"type": schema["values"], "inputBinding": b2},
+                            item, lead_pos=n, tail_pos=tail_pos))
                 binding = None
 
             if schema["type"] == "array":
@@ -75,8 +119,10 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
                     if binding:
                         b2 = copy.deepcopy(binding)
                         b2["valueFrom"] = item
-                    bindings.extend(self.bind_input({"type": schema["items"], "inputBinding": b2},
-                                                    item, lead_pos=n, tail_pos=tail_pos))
+                    bindings.extend(
+                        self.bind_input(
+                            {"type": schema["items"], "inputBinding": b2},
+                            item, lead_pos=n, tail_pos=tail_pos))
                 binding = None
 
             if schema["type"] == "File":
@@ -90,11 +136,16 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
                         datum["secondaryFiles"] = []
                     for sf in aslist(schema["secondaryFiles"]):
                         if isinstance(sf, dict) or "$(" in sf or "${" in sf:
-                            sfpath = self.do_eval(sf, context=datum)
-                            if isinstance(sfpath, basestring):
-                                sfpath = {"path": sfpath, "class": "File"}
+                            secondary_eval = self.do_eval(sf, context=datum)
+                            if isinstance(secondary_eval, basestring):
+                                sfpath = {
+                                    "path": secondary_eval, "class": "File"}
+                            else:
+                                sfpath = secondary_eval
                         else:
-                            sfpath = {"path": substitute(datum["path"], sf), "class": "File"}
+                            sfpath = {
+                                "path": substitute(datum["path"], sf),
+                                "class": "File"}
                         if isinstance(sfpath, list):
                             datum["secondaryFiles"].extend(sfpath)
                         else:
@@ -110,28 +161,22 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
 
         return bindings
 
-    def tostr(self, value):
-        if isinstance(value, dict) and value.get("class") == "File":
-            if "path" not in value:
-                raise WorkflowException("File object must have \"path\": %s" % (value))
-            return value["path"]
-        else:
-            return str(value)
-
-    def generate_arg(self, binding):
+    def generate_arg(self, binding):  # type: (Dict[str,Any]) -> List[str]
         value = binding["valueFrom"]
         if "do_eval" in binding:
             value = self.do_eval(binding["do_eval"], context=value)
 
         prefix = binding.get("prefix")
         sep = binding.get("separate", True)
 
-        l = []
+        l = []  # type: List[Dict[str,str]]
         if isinstance(value, list):
             if binding.get("itemSeparator"):
-                l = [binding["itemSeparator"].join([self.tostr(v) for v in value])]
+                l = [binding["itemSeparator"].join(
+                    [tostr(v) for v in value])]
             elif binding.get("do_eval"):
-                value = [v["path"] if isinstance(v, dict) and v.get("class") == "File" else v for v in value]
+                value = [v["path"] if isinstance(v, dict) and v.get(
+                    "class") == "File" else v for v in value]
                 return ([prefix] if prefix else []) + value
             elif prefix:
                 return [prefix]
@@ -151,13 +196,14 @@ def generate_arg(self, binding):
         args = []
         for j in l:
             if sep:
-                args.extend([prefix, self.tostr(j)])
+                args.extend([prefix, tostr(j)])
             else:
-                args.append(prefix + self.tostr(j))
+                args.append(prefix + tostr(j))
 
         return [a for a in args if a is not None]
 
     def do_eval(self, ex, context=None, pull_image=True):
+        # type: (Dict[str,str], Any, bool) -> basestring
         return expression.do_eval(ex, self.job, self.requirements,
                                   self.outdir, self.tmpdir,
                                   self.resources,