add type hints

Author: mr-c

cwltool/aslist.py

@@ -1,18 +1,23 @@
 import copy
-from aslist import aslist
-import expression
+from .utils import aslist
+from . import expression
 import avro
 import schema_salad.validate as validate
+from typing import Any, Union, AnyStr, Callable
+from .errors import WorkflowException
+from .stdfsaccess import StdFsAccess
+from .pathmapper import PathMapper
 
 CONTENT_LIMIT = 64 * 1024
 
-def substitute(value, replace):
+
+def substitute(value, replace):  # type: (str, str) -> str
     if replace[0] == "^":
         return substitute(value[0:value.rindex('.')], replace[1:])
     else:
         return value + replace
 
-def adjustFileObjs(rec, op):
+def adjustFileObjs(rec, op):  # type: (Any, Callable[[Any], Any]) -> None
     """Apply an update function to each File object in the object `rec`."""
 
     if isinstance(rec, dict):
@@ -26,9 +31,24 @@ def adjustFileObjs(rec, op):
 
 class Builder(object):
 
+    def __init__(self):  # type: () -> None
+        self.names = None  # type: avro.schema.Names
+        self.schemaDefs = None  # type: Dict[str,Dict[str,str]]
+        self.files = None  # type: List[str]
+        self.fs_access = None  # type: StdFsAccess
+        self.job = None  # type: Dict[str,str]
+        self.requirements = None  # type: List[Dict[str,Any]]
+        self.outdir = None  # type: str
+        self.tmpdir = None  # type: str
+        self.resources = None  # type: Dict[str,str]
+        self.bindings = []  # type: List[Dict[str,str]]
+        self.timeout = None  # type: int
+        self.pathmapper = None # type: PathMapper
+
     def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
-        bindings = []
-        binding = None
+        # type: (Dict[str,Any], Any, List[int], List[int]) -> List[Dict[str,str]]
+        bindings = []  # type: List[Dict[str,str]]
+        binding = None  # type: Dict[str,Any]
         if "inputBinding" in schema and isinstance(schema["inputBinding"], dict):
             binding = copy.copy(schema["inputBinding"])
 
@@ -102,9 +122,11 @@ def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]):
                         datum["secondaryFiles"] = []
                     for sf in aslist(schema["secondaryFiles"]):
                         if isinstance(sf, dict) or "$(" in sf or "${" in sf:
-                            sfpath = self.do_eval(sf, context=datum)
-                            if isinstance(sfpath, basestring):
+                            secondary_eval = self.do_eval(sf, context=datum)
+                            if isinstance(secondary_eval, basestring):
                                 sfpath = {"path": sfpath, "class": "File"}
+                            else:
+                                sfpath = secondary_eval
                         else:
                             sfpath = {"path": substitute(datum["path"], sf), "class": "File"}
                         if isinstance(sfpath, list):
@@ -126,23 +148,23 @@ def _capture_files(f):
 
         return bindings
 
-    def tostr(self, value):
+    def tostr(self, value):  # type(Any) -> str
         if isinstance(value, dict) and value.get("class") == "File":
             if "path" not in value:
                 raise WorkflowException(u"File object must have \"path\": %s" % (value))
             return value["path"]
         else:
             return str(value)
 
-    def generate_arg(self, binding):
+    def generate_arg(self, binding):  # type: (Dict[str,Any]) -> List[str]
         value = binding["valueFrom"]
         if "do_eval" in binding:
             value = self.do_eval(binding["do_eval"], context=value)
 
         prefix = binding.get("prefix")
         sep = binding.get("separate", True)
 
-        l = []
+        l = []  # type: List[Dict[str,str]]
         if isinstance(value, list):
             if binding.get("itemSeparator"):
                 l = [binding["itemSeparator"].join([self.tostr(v) for v in value])]
@@ -174,6 +196,7 @@ def generate_arg(self, binding):
         return [a for a in args if a is not None]
 
     def do_eval(self, ex, context=None, pull_image=True):
+        # type: (Dict[str,str], Any, bool) -> Any
         return expression.do_eval(ex, self.job, self.requirements,
                                   self.outdir, self.tmpdir,
                                   self.resources,

cwltool/builder.py

@@ -2,8 +2,10 @@
 import urlparse
 from rdflib import Graph, plugin, URIRef
 from rdflib.serializer import Serializer
+from typing import Any, Union, Dict, IO
 
 def makerdf(workflow, wf, ctx):
+    # type: (str, Dict[str,Any], Dict[str,Union[str, Dict[str,str]]]) -> Graph
     prefixes = {}
     for k,v in ctx.iteritems():
         if isinstance(v, dict):
@@ -26,17 +28,18 @@ def makerdf(workflow, wf, ctx):
     return g
 
 def printrdf(workflow, wf, ctx, sr, stdout):
+    # type: (str, Dict[str,Any], Dict[str,Union[str, Dict[str,str]]], str, IO[Any]) -> None
     stdout.write(makerdf(workflow, wf, ctx).serialize(format=sr))
 
-def lastpart(uri):
+def lastpart(uri):  # type: (Any) -> str
     uri = str(uri)
     if "/" in uri:
         return uri[uri.rindex("/")+1:]
     else:
         return uri
 
 
-def dot_with_parameters(g, stdout):
+def dot_with_parameters(g, stdout):  # type: (Graph, IO[Any]) -> None
     qres = g.query(
         """SELECT ?step ?run ?runtype
            WHERE {
@@ -92,8 +95,8 @@ def dot_with_parameters(g, stdout):
     for (inp,) in qres:
         stdout.write(u'"%s" [shape=octagon]\n' % (lastpart(inp)))
 
-def dot_without_parameters(g, stdout):
-    dotname = {}
+def dot_without_parameters(g, stdout):  # type: (Graph, IO[Any]) -> None
+    dotname = {}  # type: Dict[str,str]
     clusternode = {}
 
     stdout.write("compound=true\n")
@@ -166,14 +169,15 @@ def dot_without_parameters(g, stdout):
 
 
 def printdot(workflow, wf, ctx, stdout, include_parameters=False):
+    # type: (str, Dict[str,Any], Dict[str,Union[str, Dict[str,str]]], Any, bool) -> None
     g = makerdf(workflow, wf, ctx)
 
     stdout.write("digraph {")
 
     #g.namespace_manager.qname(predicate)
 
     if include_parameters:
-        dot_with_parmeters(g, stdout)
+        dot_with_parameters(g, stdout)
     else:
         dot_without_parameters(g, stdout)
 

cwltool/cwlrdf.py

@@ -8,9 +8,11 @@
 import shutil
 import tempfile
 import yaml
+import yaml.scanner
 import pipes
 import logging
 import schema_salad.ref_resolver
+from typing import Any, Union
 
 _logger = logging.getLogger("cwltest")
 _logger.addHandler(logging.StreamHandler())
@@ -21,7 +23,8 @@
 class CompareFail(Exception):
     pass
 
-def compare(a, b):
+
+def compare(a, b):  # type: (Any, Any) -> bool
     try:
         if isinstance(a, dict):
             if a.get("class") == "File":
@@ -54,8 +57,9 @@ def compare(a, b):
     except Exception as e:
         raise CompareFail(str(e))
 
-def run_test(args, i, t):
-    out = {}
+
+def run_test(args, i, t):  # type: (argparse.Namespace, Any, Dict[str,str]) -> int
+    out = {}  # type: Dict[str,Any]
     outdir = None
     try:
         if "output" in t:
@@ -84,7 +88,7 @@ def run_test(args, i, t):
             outstr = subprocess.check_output(test_command)
             out = yaml.load(outstr)
     except ValueError as v:
-        _logger.error(v)
+        _logger.error(str(v))
         _logger.error(outstr)
     except subprocess.CalledProcessError as err:
         if err.returncode == UNSUPPORTED_FEATURE:
@@ -123,15 +127,16 @@ def run_test(args, i, t):
             failed = True
 
     if outdir:
-        shutil.rmtree(outdir, True)
+        shutil.rmtree(outdir, True)  # type: ignore
+        # Weird AnyStr != basestring issue
 
     if failed:
         return 1
     else:
         return 0
 
 
-def main():
+def main():  # type: () -> int
     parser = argparse.ArgumentParser(description='Compliance tests for cwltool')
     parser.add_argument("--test", type=str, help="YAML file describing test cases", required=True)
     parser.add_argument("--basedir", type=str, help="Basedir to use for tests", default=".")

cwltool/cwltest.py

@@ -3,13 +3,15 @@
 import sys
 import requests
 import os
-import process
+from .errors import WorkflowException
 import re
 import tempfile
+from typing import Any, Union
 
 _logger = logging.getLogger("cwltool")
 
 def get_image(dockerRequirement, pull_image, dry_run=False):
+    # type: (Dict[str,str], bool, bool) -> bool
     found = False
 
     if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
@@ -36,7 +38,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
                 subprocess.check_call(cmd, stdout=sys.stderr)
                 found = True
         elif "dockerFile" in dockerRequirement:
-            dockerfile_dir = tempfile.mkdtemp()
+            dockerfile_dir = str(tempfile.mkdtemp())
             with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as df:
                 df.write(dockerRequirement["dockerFile"])
             cmd = ["docker", "build", "--tag=%s" % dockerRequirement["dockerImageId"], dockerfile_dir]
@@ -64,7 +66,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
                     loadproc.stdin.close()
                 rcode = loadproc.wait()
                 if rcode != 0:
-                    raise process.WorkflowException("Docker load returned non-zero exit status %i" % (rcode))
+                    raise WorkflowException("Docker load returned non-zero exit status %i" % (rcode))
                 found = True
         elif "dockerImport" in dockerRequirement:
             cmd = ["docker", "import", dockerRequirement["dockerImport"], dockerRequirement["dockerImageId"]]
@@ -77,6 +79,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
 
 
 def get_from_requirements(r, req, pull_image, dry_run=False):
+    # type: (Dict[str,str], bool, bool, bool) -> Union[None,str]
     if r:
         errmsg = None
         try:
@@ -88,14 +91,14 @@ def get_from_requirements(r, req, pull_image, dry_run=False):
 
         if errmsg:
             if req:
-                raise process.WorkflowException(errmsg)
+                raise WorkflowException(errmsg)
             else:
                 return None
 
         if get_image(r, pull_image, dry_run):
             return r["dockerImageId"]
         else:
             if req:
-                raise process.WorkflowException(u"Docker image %s not found" % r["dockerImageId"])
+                raise WorkflowException(u"Docker image %s not found" % r["dockerImageId"])
 
     return None

cwltool/docker.py

@@ -1,7 +1,8 @@
 import subprocess
+from typing import Union
 
 
-def docker_vm_uid():
+def docker_vm_uid():  # type: () -> Union[int,None]
     """
     Returns the UID of the default docker user inside the VM
 
@@ -19,7 +20,7 @@ def docker_vm_uid():
         return None
 
 
-def check_output_and_strip(cmd):
+def check_output_and_strip(cmd):  # type: (List[str]) -> Union[str,None]
     """
     Passes a command list to subprocess.check_output, returning None
     if an expected exception is raised
@@ -36,7 +37,7 @@ def check_output_and_strip(cmd):
         return None
 
 
-def docker_machine_name():
+def docker_machine_name():  # type: () -> Union[str,None]
     """
     Get the machine name of the active docker-machine machine
     :return: Name of the active machine or None if error
@@ -45,6 +46,7 @@ def docker_machine_name():
 
 
 def cmd_output_matches(check_cmd, expected_status):
+    # type: (List[str], str) -> bool
     """
     Runs a command and compares output to expected
     :param check_cmd: Command list to execute
@@ -57,15 +59,15 @@ def cmd_output_matches(check_cmd, expected_status):
         return False
 
 
-def boot2docker_running():
+def boot2docker_running():  # type: () -> bool
     """
     Checks if boot2docker CLI reports that boot2docker vm is running
     :return: True if vm is running, False otherwise
     """
     return cmd_output_matches(['boot2docker', 'status'], 'running')
 
 
-def docker_machine_running():
+def docker_machine_running():  # type: () -> bool
     """
     Asks docker-machine for active machine and checks if its VM is running
     :return: True if vm is running, False otherwise
@@ -74,7 +76,7 @@ def docker_machine_running():
     return cmd_output_matches(['docker-machine', 'status', machine_name], 'Running')
 
 
-def cmd_output_to_int(cmd):
+def cmd_output_to_int(cmd):  # type: (List[str]) -> Union[int,None]
     """
     Runs the provided command and returns the integer value of the result
     :param cmd: The command to run
@@ -83,22 +85,21 @@ def cmd_output_to_int(cmd):
     result = check_output_and_strip(cmd)  # may return None
     if result is not None:
         try:
-            result = int(result)
+            return int(result)
         except ValueError:
             # ValueError is raised if int conversion fails
-            result = None
-    return result
+            return None
 
 
-def boot2docker_uid():
+def boot2docker_uid():  # type: () -> Union[int,None]
     """
     Gets the UID of the docker user inside a running boot2docker vm
     :return: the UID, or None if error (e.g. boot2docker not present or stopped)
     """
     return cmd_output_to_int(['boot2docker', 'ssh', 'id', '-u'])
 
 
-def docker_machine_uid():
+def docker_machine_uid():  # type: () -> Union[int,None]
     """
     Asks docker-machine for active machine and gets the UID of the docker user
     inside the vm