Docker socket

cwltool/job.py

@@ -55,6 +55,18 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, rm_tmpdir=True,
             env = os.environ
             img_id = docker.get_from_requirements(docker_req, docker_is_req, pull_image)
 
+        (docker_socket_req, _) = get_feature(self, "DockerSocketRequirement")
+
+        gid = os.getgid()
+
+        if docker_socket_req:
+            if kwargs.get("enable_docker_socket") is not True:
+                raise WorkflowException("DockerSocketRequirement is present but enable_docker_socket is not True")
+            if docker_socket_req.get("additionalDockerImages"):
+                for d in docker_socket_req["additionalDockerImages"]:
+                    docker.get_from_requirements(docker_req, True, pull_image)
+            gid = os.stat("/var/run/docker.sock").st_gid
+
         if docker_is_req and img_id is None:
             raise WorkflowException("Docker is required for running this tool.")
 
@@ -67,7 +79,7 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, rm_tmpdir=True,
             runtime.append("--volume=%s:%s:rw" % (os.path.abspath(self.tmpdir), "/tmp/job_tmp"))
             runtime.append("--workdir=%s" % ("/tmp/job_output"))
             euid = docker_vm_uid() or os.geteuid()
-            runtime.append("--user=%s" % (euid))
+            runtime.append("--user=%s:%s" % (euid, gid))
 
             if rm_container:
                 runtime.append("--rm")
@@ -77,6 +89,9 @@ def run(self, dry_run=False, pull_image=True, rm_container=True, rm_tmpdir=True,
             for t,v in self.environment.items():
                 runtime.append("--env=%s=%s" % (t, v))
 
+            if docker_socket_req:
+                runtime.append("--volume=/var/run/docker.sock:/var/run/docker.sock:rw")
+
             runtime.append(img_id)
         else:
             env = self.environment

cwltool/main.py

@@ -77,7 +77,14 @@ def arg_parser():
                         dest="move_outputs")
 
     exgroup = parser.add_mutually_exclusive_group()
-    exgroup.add_argument("--enable-pull", default=True, action="store_true",
+    exgroup.add_argument("--enable-docker-socket", default=False, action="store_true",
+                        help="Permit DockerSocketRequirement", dest="enable_docker_socket")
+
+    exgroup.add_argument("--disable-docker-socket", default=False, action="store_false",
+                        help="Disallow DockerSocketRequirement", dest="enable_docker_socket")
+
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--enable-", default=True, action="store_true",
                         help="Try to pull Docker images", dest="enable_pull")
 
     exgroup.add_argument("--disable-pull", default=True, action="store_false",
@@ -431,7 +438,8 @@ def main(args=None,
                        tmpdir_prefix=args.tmpdir_prefix,
                        rm_tmpdir=args.rm_tmpdir,
                        makeTool=makeTool,
-                       move_outputs=args.move_outputs
+                       move_outputs=args.move_outputs,
+                       enable_docker_socket=args.enable_docker_socket
                        )
         # This is the workflow output, it needs to be written
         stdout.write(json.dumps(out, indent=4))

cwltool/process.py

@@ -28,7 +28,8 @@
                                 "CreateFileRequirement",
                                 "ScatterFeatureRequirement",
                                 "SubworkflowFeatureRequirement",
-                                "MultipleInputFeatureRequirement"]
+                                "MultipleInputFeatureRequirement",
+                                "DockerSocketRequirement"]
 
 def get_schema():
     f = resource_stream(__name__, 'schemas/draft-3/cwl-avro.yml')
@@ -167,6 +168,8 @@ def _init_job(self, joborder, input_basedir, **kwargs):
         for r in self.requirements:
             if r["class"] not in supportedProcessRequirements:
                 raise WorkflowException("Unsupported process requirement %s" % (r["class"]))
+            if r["class"] == "DockerSocketRequirement" and kwargs.get("enable_docker_socket") is not True:
+                raise WorkflowException("DockerSocketRequirement is present but enable_docker_socket is not True")
 
         builder.files = []
         builder.bindings = []

cwltool/schemas/draft-2/draft-2/dndtool.cwl

@@ -0,0 +1,29 @@
+class: CommandLineTool
+description: "Reverse each line using the `rev` command"
+reqirements:
+  - class: DockerRequirement
+    dockerId: docker_in_docker
+    dockerFile: |
+      FROM ubuntu:14.04
+      MAINTAINER peter.amstutz@curoverse.com
+
+      RUN apt-get update -qq && apt-get install -qqy \
+      apt-transport-https \
+      ca-certificates \
+      curl \
+      lxc \
+      iptables \
+      python-setuptools
+
+      # Install Docker from Docker Inc. repositories.
+      RUN curl -sSL https://get.docker.com/ubuntu/ | sh
+
+  - class: DockerSocketRequirement
+inputs: []
+outputs:
+  - id: "#output"
+    type: File
+    outputBinding:
+      glob: output.txt
+baseCommand: [docker, version]
+stdout: output.txt

cwltool/schemas/draft-3/cwl-avro.yml

@@ -2044,3 +2044,29 @@
         the expression engine.  The semantics of this field are defined by the
         underlying expression engine.  Intended for uses such as providing
         function definitions that will be called from CWL expressions.
+
+
+- name: DockerSocketRequirement
+  type: record
+  extends: "#ProcessRequirement"
+  fields:
+    - name: additionalDockerImages
+      type:
+        - "null"
+        - type: array
+          items: "#DockerRequirement"
+  doc: |
+    Require that the command should have access to the Docker socket at
+    `/var/run/docker.sock`. If the command is itself run in Docker, this means
+    the Docker socket must be made available inside the Docker container
+    through a bind mount.
+
+    If this requirement is present, the platform is strongly encouraged to
+    arrange that file paths are consistent inside and outside the container, as
+    paths that exist only inside the container cannot be passed to other
+    containers.
+
+    Note there are inherent security problems with providing access to the
+    Docker socket, so use of this feature should be limited to trusted
+    environments running trusted workflows. This feature is provided to support
+    existing container schemes that rely on Docker-in-Docker functionality.

cwltool/schemas/draft-3/draft-3/add-lines-wf.cwl

@@ -1 +1 @@
-../draft-2/add-lines-wf.cwl
+../../draft-2/draft-2/add-lines-wf.cwl

cwltool/schemas/draft-3/draft-3/binding-test.cwl

@@ -1 +1 @@
-../draft-2/binding-test.cwl
+../../draft-2/draft-2/binding-test.cwl

cwltool/schemas/draft-3/draft-3/bwa-mem-job.json

@@ -1 +1 @@
-../draft-2/bwa-mem-job.json
+../../draft-2/draft-2/bwa-mem-job.json

cwltool/schemas/draft-3/draft-3/cat-job.json

@@ -1 +1 @@
-../draft-2/cat-job.json
+../../draft-2/draft-2/cat-job.json

cwltool/schemas/draft-3/draft-3/cat-n-job.json

@@ -1 +1 @@
-../draft-2/cat-n-job.json
+../../draft-2/draft-2/cat-n-job.json

cwltool/schemas/draft-3/draft-3/cat1-tool.cwl

@@ -1 +1 @@
-../draft-2/cat1-tool.cwl
+../../draft-2/draft-2/cat1-tool.cwl

cwltool/schemas/draft-3/draft-3/cat2-tool.cwl

@@ -1 +1 @@
-../draft-2/cat2-tool.cwl
+../../draft-2/draft-2/cat2-tool.cwl

cwltool/schemas/draft-3/draft-3/cat3-tool.cwl

@@ -1 +1 @@
-../draft-2/cat3-tool.cwl
+../../draft-2/draft-2/cat3-tool.cwl

cwltool/schemas/draft-3/draft-3/cat4-tool.cwl

@@ -1 +1 @@
-../draft-2/cat4-tool.cwl
+../../draft-2/draft-2/cat4-tool.cwl

cwltool/schemas/draft-3/draft-3/cat5-tool.cwl

@@ -1 +1 @@
-../draft-2/cat5-tool.cwl
+../../draft-2/draft-2/cat5-tool.cwl

cwltool/schemas/draft-3/draft-3/count-lines3-job.json

@@ -1 +1 @@
-../draft-2/count-lines3-job.json
+../../draft-2/draft-2/count-lines3-job.json

cwltool/schemas/draft-3/draft-3/count-lines4-job.json

@@ -1 +1 @@
-../draft-2/count-lines4-job.json
+../../draft-2/draft-2/count-lines4-job.json

cwltool/schemas/draft-3/draft-3/count-lines6-job.json

@@ -1 +1 @@
-../draft-2/count-lines6-job.json
+../../draft-2/draft-2/count-lines6-job.json

cwltool/schemas/draft-3/draft-3/echo-tool.cwl

@@ -1 +1 @@
-../draft-2/echo-tool.cwl
+../../draft-2/draft-2/echo-tool.cwl

cwltool/schemas/draft-3/draft-3/empty.json

@@ -1 +1 @@
-../draft-2/empty.json
+../../draft-2/draft-2/empty.json

cwltool/schemas/draft-3/draft-3/env-job.json

@@ -1 +1 @@
-../draft-2/env-job.json
+../../draft-2/draft-2/env-job.json

cwltool/schemas/draft-3/draft-3/env-tool1.cwl

@@ -1 +1 @@
-../draft-2/env-tool1.cwl
+../../draft-2/draft-2/env-tool1.cwl

cwltool/schemas/draft-3/draft-3/env-tool2.cwl

@@ -1 +1 @@
-../draft-2/env-tool2.cwl
+../../draft-2/draft-2/env-tool2.cwl

cwltool/schemas/draft-3/draft-3/hello.txt

@@ -1 +1 @@
-../draft-2/hello.txt
+../../draft-2/draft-2/hello.txt

cwltool/schemas/draft-3/draft-3/index.py

@@ -1 +1 @@
-../draft-2/index.py
+../../draft-2/draft-2/index.py

cwltool/schemas/draft-3/draft-3/node-engine.cwl

@@ -1 +1 @@
-../draft-2/node-engine.cwl
+../../draft-2/draft-2/node-engine.cwl

cwltool/schemas/draft-3/draft-3/number.txt

@@ -1 +1 @@
-../draft-2/number.txt
+../../draft-2/draft-2/number.txt

cwltool/schemas/draft-3/draft-3/parseInt-job.json

@@ -1 +1 @@
-../draft-2/parseInt-job.json
+../../draft-2/draft-2/parseInt-job.json

cwltool/schemas/draft-3/draft-3/rename-job.json

@@ -1 +1 @@
-../draft-2/rename-job.json
+../../draft-2/draft-2/rename-job.json

cwltool/schemas/draft-3/draft-3/rename.cwl

@@ -1 +1 @@
-../draft-2/rename.cwl
+../../draft-2/draft-2/rename.cwl

cwltool/schemas/draft-3/draft-3/revsort-job.json

@@ -1 +1 @@
-../draft-2/revsort-job.json
+../../draft-2/draft-2/revsort-job.json

cwltool/schemas/draft-3/draft-3/revtool.cwl

@@ -1 +1 @@
-../draft-2/revtool.cwl
+../../draft-2/draft-2/revtool.cwl

cwltool/schemas/draft-3/draft-3/scatter-job1.json

@@ -1 +1 @@
-../draft-2/scatter-job1.json
+../../draft-2/draft-2/scatter-job1.json

cwltool/schemas/draft-3/draft-3/scatter-job2.json

@@ -1 +1 @@
-../draft-2/scatter-job2.json
+../../draft-2/draft-2/scatter-job2.json

cwltool/schemas/draft-3/draft-3/scatter-wf1.cwl

@@ -1 +1 @@
-../draft-2/scatter-wf1.cwl
+../../draft-2/draft-2/scatter-wf1.cwl

cwltool/schemas/draft-3/draft-3/scatter-wf2.cwl

@@ -1 +1 @@
-../draft-2/scatter-wf2.cwl
+../../draft-2/draft-2/scatter-wf2.cwl

cwltool/schemas/draft-3/draft-3/search-job.json

@@ -1 +1 @@
-../draft-2/search-job.json
+../../draft-2/draft-2/search-job.json

cwltool/schemas/draft-3/draft-3/search.py

@@ -1 +1 @@
-../draft-2/search.py
+../../draft-2/draft-2/search.py

cwltool/schemas/draft-3/draft-3/sorttool.cwl

@@ -1 +1 @@
-../draft-2/sorttool.cwl
+../../draft-2/draft-2/sorttool.cwl

cwltool/schemas/draft-3/draft-3/tmap-job.json

@@ -1 +1 @@
-../draft-2/tmap-job.json
+../../draft-2/draft-2/tmap-job.json

cwltool/schemas/draft-3/draft-3/underscore.js

@@ -1 +1 @@
-../draft-2/underscore.js
+../../draft-2/draft-2/underscore.js

cwltool/schemas/draft-3/draft-3/wc-job.json

@@ -1 +1 @@
-../draft-2/wc-job.json
+../../draft-2/draft-2/wc-job.json

cwltool/schemas/draft-3/draft-3/wc-tool.cwl

@@ -1 +1 @@
-../draft-2/wc-tool.cwl
+../../draft-2/draft-2/wc-tool.cwl

cwltool/schemas/draft-3/draft-3/whale.txt

@@ -1 +1 @@
-../draft-2/whale.txt
+../../draft-2/draft-2/whale.txt