Squashed 'cwltool/schemas/' changes from 2cf0896..7a9bee2

7a9bee2 Add 'v1.1.0-dev1/salad/' from commit '98a5bea104672f9c652ad17ead5c6869a0da2cdf'
5853de0 refresh schema salad
762d534 update new draft documentation
a33dd70 initial v1.1.0-dev1
2c85ac4 README update
3a5061d Add support for --junit-xml to run_test.sh
d37ef31 Run tests with "cwltest" instead of "python -m cwltool.cwltest"
cf28454 Add test for using initial work dir, docker, and secondary files.
98a5bea Update py3 setuptools pip wheel
8646e82 Run install-dep for py3
d866c53 pin ruamel py3 install
04dd72f Add ruamel.yaml to py3 jenkins testing.
56ddb9d Update requirements.txt again for CacheControl and lockfile.
a5ac105 Update requirements.txt
5dff7db Add version constraint typing >= 3.5.2 so ruamel typechecks work. (#55)
6fb2969 Merge pull request #54 from common-workflow-language/newer-ruamel
bc57aa0 adopt fixed ruamel.yaml
236b042 make mypy invocation py3 compat
61636e4 switch to latest ruamel.yaml with native types
fefc957 install typed-ast during Jenkins tests
c570b8a update to minimally required version of pip
bada048 remove old setuptools wheel, then fix pip, then upgrade setuptools
454733c fix pip, then install setuptools
a0c7d1a fix pip
9012e2b forcibly remove old setuptools
559f19a fix setuptools version specification typo
8a34425 update release test
a833808 Merge pull request #52 from common-workflow-language/install-tests
ac7341c last fix for the release test!
70dbadc also ship metaschema in source distribution
ad0b913 add the Makefile to the source distribution
6df505c clarify module != package name
5890eb2 make tests installable and runnable after install
0514920 Merge pull request #51 from common-workflow-language/tighten-typecheck
caa854a tweak travis/tox config
b0c357b tighten up types
ab9f395 Merge pull request #50 from common-workflow-language/type-docs
4ef0ed1 document how to expand Typeshed
c646a62 Merge pull request #49 from common-workflow-language/schema-cache
14d51ca fix types
7dbffaf Add general http caching support.  Applies to $import, $include and $schemas.
f8297cc Cache external schemas after being loaded 1st time (doesn't cache to disk, though.)
2300cb0 Merge pull request #48 from common-workflow-language/fix-jsonld-context
449b5b8 Bump version
c8305c0 Add default @id field to json-ld context if missing.
ea983fd Merge pull request #47 from common-workflow-language/install-tests
7bd46e1 make tests installable and runnable after install
66b5ac6 update ruamel.yaml types
9e11a86 Merge pull request #46 from common-workflow-language/ruamel-yaml-bump
17fe62e re-pin ruamel.yaml
fe7c49f Merge pull request #45 from common-workflow-language/improve-makerdf
3b414e7 replace variable shadowing with more meaningful names
1b1f58f replace Py3 exception passing to logger with py2 version
b8ba072 update types
7008d94 Bugfix makerdf (helper function for converting to RDF).
94d9847 Merge pull request #44 from common-workflow-language/revert-optional-required
8a211bc Revert "make explicit optional and required fields"
c9fc39f Name field is ambigous, so use list.
f814cdb Can't distinguish between single SpecializeDef and mapSubject/mapPredicate, so disallow.
9f3927d update to v1.14
13061e5 Merge pull request #42 from common-workflow-language/specialize-refscope
750922d Can use refScope and mapSubject/mapPredicate on specialize.
9773208 Merge pull request #38 from common-workflow-language/mixin
5d1bda4 Merge pull request #40 from common-workflow-language/doc-map
a6d6136 update type for typefmt()
41f414d Merge pull request #41 from common-workflow-language/rdfscomment
c3ada96 update type for fetch()
2abc7f4 Doc generation understands mapSubject/mapPredicate -> map<>
db8f409 sld:doc -> rdfs:comment
aea5636 Add documentation about mixin.
63822d6 Remove spurious prints.
4217af8 Add mixin
d626a74 Implement $mixin
93ed1fb pin ruamel.yaml to working version
aa83430 Merge pull request #36 from common-workflow-language/explicit-optional
a58b955 make explicit optional and required fields
c509a07 Merge pull request #34 from common-workflow-language/unscoped_ids
f857e9d type fixups
0407370 Allow idmaps for record fields.
5d9a08e Use makerdf for --print-rdf
ce00252 Add test for non-scoped identifiers.
ef27301 restore a dropped cast

git-subtree-dir: cwltool/schemas
git-subtree-split: 7a9bee2

Author: mr-c

cwltool/schemas/README.md

@@ -1,94 +1,80 @@
 Common Workflow Language
 ========================
 
-The Common Workflow Language (CWL) is an informal, multi-vendor working group
-consisting of various organizations and individuals that have an interest in
-portability of data analysis workflows.  Our goal is to create specifications
-that enable data scientists to describe analysis tools and workflows that are
-powerful, easy to use, portable, and support reproducibility.
+[![GitHub stars](https://img.shields.io/github/stars/common-workflow-language/common-workflow-language.svg)](https://github.com/common-workflow-language/common-workflow-language/stargazers) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/common-workflow-language/common-workflow-language?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-CWL builds on technologies such as [JSON-LD](http://json-ld.org)
-for data modeling and [Docker](http://docker.com) for portable runtime\
-environments.
-
-CWL is designed to express workflows for data-intensive science, such as
-Bioinformatics, Medical Imaging, Chemistry, Physics, and Astronomy.
+The Common Workflow Language (CWL) is a specification for describing analysis
+workflows and tools that are portable and scalable across a variety of software
+and hardware environments, from workstations to cluster, cloud, and high
+performance computing (HPC) environments.  CWL is designed to meet the needs of
+data-intensive science, such as Bioinformatics, Medical Imaging, Astronomy,
+Physics, and Chemistry.
 
-## CWL User Guide
+CWL is developed by an informal, multi-vendor working group consisting of
+organizations and individuals interested enabling scientists to share data
+analysis workflows.  [The CWL project is on Github.](https://github.com/common-workflow-language/common-workflow-language)
 
-[User guide for the current stable specification (v1.0)](http://www.commonwl.org/v1.0/UserGuide.html),
-provides a gentle introduction to writing CWL command line tool and workflow descriptions.
+CWL builds on technologies such as [JSON-LD](http://json-ld.org)
+for data modeling and [Docker](http://docker.com) for portable runtime
+environments.
 
-## CWL Specification
+## User Guide
 
-The current stable specification is [v1.0](http://www.commonwl.org/v1.0/):
+The [CWL user guide (v1.0)](http://www.commonwl.org/v1.0/UserGuide.html) provides a
+gentle introduction to learning how to write CWL command line tool and workflow
+descriptions.
 
-http://www.commonwl.org/v1.0/
+## Specification
 
-Older drafts: [draft-1](https://github.com/common-workflow-language/common-workflow-language/tree/master/draft-1), [draft-2](http://www.commonwl.org/draft-2/), [draft-3](http://www.commonwl.org/draft-3/)
+For developers and advanced users, the current
+[CWL specification (v1.0)](http://www.commonwl.org/v1.0/) provides
+authoritative documentation of the execution of CWL documents.  Links older
+drafts:
+[draft-1](https://github.com/common-workflow-language/common-workflow-language/tree/master/draft-1),
+[draft-2](http://www.commonwl.org/draft-2/),
+[draft-3](http://www.commonwl.org/draft-3/)
 
 ### Citation
 
 Please cite https://dx.doi.org/10.6084/m9.figshare.3115156.v2
 
 ## Implementations
 
-Some of the software supporting running Common Workflow Language tools or workflows:
-
-* [cwltool (reference implementation)](https://github.com/common-workflow-language/cwltool),
-  [cwltool wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/cwltool-%28reference-implementation%29)
-* [Rabix](https://github.com/rabix/rabix),
-  [Rabix wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Rabix)
-* [Arvados](https://arvados.org),
-  [Arvados wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Arvados)
-* [Galaxy](https://github.com/common-workflow-language/Galaxy),
-  [Galaxy wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Galaxy)
-* [Parallel Recipes](https://github.com/yvdriess/precipes),
-  [Parallel Recipes wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Parallel-Recipes)
-* [Toil](https://github.com/BD2KGenomics/toil),
-  [Toil wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Toil)
-* [CancerCollaboratory](https://github.com/CancerCollaboratory),
-  [CancerCollaboratory wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/CancerCollaboratory)
-* [Airflow (SciDAP)](https://github.com/SciDAP/scidap),
-  [Airflow wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/SciDAP)
-* [cwl2script](https://github.com/common-workflow-language/cwl2script),
-  [cwl2script page](https://github.com/common-workflow-language/common-workflow-language/wiki/cwl2script)
-* [Apache Taverna](http://taverna.incubator.apache.org/),
-  [Apache Taverna wiki page](https://github.com/common-workflow-language/common-workflow-language/wiki/Taverna)
-
-We continuously run the CWL conformance tests on several implementations:
-
-https://ci.commonwl.org
-
-## Examples
-
-[Github repository of example tools and workflows.](https://github.com/common-workflow-language/workflows)
-
-## Support
-
-The best place to ask a question about all things CWL is on
+|Software|Description|CWL support|Platform support|Maturity|
+|--------|-----------|-----------|--------|--------|
+|[cwltool](https://github.com/common-workflow-language/cwltool)|Reference implementation of CWL|[![Build Status](https://ci.commonwl.org/job/cwltool-conformance/badge/icon)](http://ci.commonwl.org/job/cwltool-conformance/lastBuild/testReport/)|Linux, OS X, local execution only|Production|
+|[Arvados](https://arvados.org)|Distributed computing platform for data analysis on massive data sets. [Using CWL on Arvados](http://doc.arvados.org/user/cwl/cwl-runner.html)|[![Build Status](https://ci.commonwl.org/job/arvados-conformance/badge/icon)](http://ci.commonwl.org/job/arvados-conformance/lastBuild/testReport/)|AWS, GCP, Azure, Slurm|Production|
+|[Toil](https://github.com/BD2KGenomics/toil)|Toil is a workflow engine entirely written in Python.|[![Build Status](https://ci.commonwl.org/job/toil-conformance/badge/icon)](http://ci.commonwl.org/job/toil-conformance/lastBuild/testReport/)|AWS, GCP, Azure, OpenStack, Grid Engine, Mesos|Production|
+|[Apache Taverna](http://taverna.incubator.apache.org/)|Domain-independent Workflow Management System|[alpha](https://issues.apache.org/jira/browse/TAVERNA-900)|Java|Production|
+|[Galaxy](https://galaxyproject.org/)|Web-based platform for data intensive biomedical research.|[alpha](https://github.com/common-workflow-language/galaxy)|Python|Production|
+|[AWE](https://github.com/MG-RAST/AWE)|Workflow and resource management system for bioinformatics data analysis.|[alpha](https://github.com/wgerlach/AWE)|Go|Production|
+|[Funnel](https://github.com/bmeg/funnel)|Use Google Genomics Pipeline API with CWL|alpha|GCP|alpha|
+|[Rabix Bunny](https://github.com/rabix/bunny)|Reproducible Analyses for Bioinformatics|alpha|Java|alpha|
+
+## Repositories of CWL Tools and Workflows
+
+|Repository|Description|
+|----|-----------|
+|[Workflows repository](https://github.com/common-workflow-language/workflows)|Git repository of user contributed tools and workflows.|
+|[Dockstore tool registry](http://dockstore.org)|An open platform for sharing Docker-based tools described with the Common Workflow Language used by the GA4GH.|
+
+## Software for working with CWL
+
+|Software|Description|
+|--------|-----------|
+|[cwltest](https://github.com/common-workflow-language/cwltest)|CWL testing framework,  automated testing of tools and workflows written with CWL|
+|[cwl-upgrader](https://github.com/common-workflow-language/cwl-upgrader)|Upgrade CWL documents from draft-3 to v1.0|
+|[argparse2cwl](https://github.com/common-workflow-language/gxargparse#argparse2cwl)|Generate CWL CommandLineTool wrapper from Python programs that use argparse.  Also supports [click](http://click.pocoo.org/5/) argument parser.|
+|[cwl2argparse](https://github.com/common-workflow-language/cwl2argparse)|Generate Python argparse code from CWL CommandLineTool description.|
+|[pypi2cwl](https://github.com/common-workflow-language/pypi2cwl)|Automatically run argparse2cwl on any package in PyPi|
+|[cwlavro](https://github.com/common-workflow-language/cwlavro)|Java classes for loading CWL documents|
+
+## Support, Community and Contributing
+
+The recommended place to ask a question about all things CWL is on
 [Biostars](https://www.biostars.org/t/cwl/).
 
-[![Biostars
-CWL](https://www.biostars.org/static/biostar2.logo.png)](https://www.biostars.org/t/cwl/)
-</a>
-
-## Development and testing
-
-[The CWL project is on Github.](https://github.com/common-workflow-language/common-workflow-language)
-
-[![GitHub
-stars](https://img.shields.io/github/stars/common-workflow-language/common-workflow-language.svg)](https://github.com/common-workflow-language/common-workflow-language/stargazers)
-
-There is a Jenkins server maintained by Curoverse that runs tests for the
-reference implementation, builds and uploads packages, and builds and uploads
-Docker images:
-
-https://ci.curoverse.com/job/common-workflow-language/
-
-Current build status: [![Build Status](https://ci.curoverse.com/buildStatus/icon?job=common-workflow-language)](https://ci.curoverse.com/job/common-workflow-language/)
-
-## Community and Contributing
+[![Biostars CWL](https://www.biostars.org/static/biostar2.logo.png)](https://www.biostars.org/t/cwl/)
 
 If you are interested in learning more or contributing ideas or code,
 [come chat with us on Gitter](https://gitter.im/common-workflow-language/common-workflow-language),
@@ -97,8 +83,6 @@ join the [mailing list common-workflow-language on Google Groups](https://groups
 [fork the repository](https://github.com/common-workflow-language/common-workflow-language)
 and send a pull request!
 
-[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/common-workflow-language/common-workflow-language?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-
 Your CWL Community Engineer, Michael R. Crusoe, publishes a blog about his work
 with updates at http://mrc.commonwl.org.
 

cwltool/schemas/doc/add-new-draft.md

@@ -1,17 +1,18 @@
 How to make a new draft version of the CWL spec
 
-1. Duplicate prior draft-n directory to draft-n+1 in a branch
+1. Duplicate prior directory to "vNewVersion-dev1" or "vExistingVersion-devN+1'
+   in a branch
 2. Update references to the new draft name.
 3. Pull in the latest metaschema, where `schema_salad_repo` is the remote
    repository for the schema salad tool.
 
      git fetch --all
-     git subtree add -P draft-4/salad schema_salad_repo/master
+     git subtree add -P v1.1.0-dev1/salad schema_salad_repo/master
 
 4. In the reference implementation (cwltool): make a new branch, and update the
    subtree checkout of the spec:
 
-     git subtree merge -P cwltool/schemas/ cwl_repo/draft-4
+     git subtree merge -P cwltool/schemas/ cwl_repo/v1.1.0-dev1
 
    Where `cwl_repo` is the remote repository for the CWL specifications.
 4. In the reference implementation, teach it about the new draft version:

cwltool/schemas/run_test.sh

@@ -13,6 +13,7 @@ EOF
 
 DRAFT=v1.0
 TEST_N=""
+JUNIT_XML=""
 RUNNER=cwl-runner
 PLATFORM=$(uname -s)
 COVERAGE="python"
@@ -35,6 +36,9 @@ do
         --only-tools)
             ONLY_TOOLS=--only-tools
             ;;
+        --junit-xml=*)
+            JUNIT_XML=$arg
+            ;;
         *=*)
             eval $(echo $arg | cut -d= -f1)=\"$(echo $arg | cut -d= -f2-)\"
             ;;
@@ -64,9 +68,9 @@ runtest() {
 
     runs=$((runs+1))
     (cd $DRAFT
-     ${COVERAGE} -m cwltool.cwltest --tool "$1" \
+     cwltest --tool "$1" \
 	     --test=conformance_test_${DRAFT}.yaml ${TEST_N} \
-	     ${TEST_L} ${ONLY_TOOLS} --basedir ${DRAFT}
+	     ${TEST_L} ${ONLY_TOOLS} ${JUNIT_XML} --basedir ${DRAFT}
     )
     checkexit
 }

cwltool/schemas/v1.0/conformance_test_v1.0.yaml

@@ -867,3 +867,19 @@
     }
   tool: "v1.0/js-expr-req-wf.cwl#wf"
   doc: Test that expressionLib requirement of individual tool step overrides expressionLib of workflow.
+
+- job: v1.0/initialworkdirrequirement-docker-out-job.json
+  output:
+    OUTPUT:
+      "checksum": "sha1$aeb3d11bdf536511649129f4077d5cda6a324118"
+      "location": "ref.fasta"
+      "secondaryFiles": [{
+        "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709",
+        "location": "ref.fasta.fai",
+          "class": "File",
+          "size": 0
+      }]
+      "class": "File"
+      "size": 12010
+  tool: v1.0/initialworkdirrequirement-docker-out.cwl
+  doc: Test output of InitialWorkDir

v1.0/v1.0/initialworkdirrequirement-docker-out-job.json

@@ -0,0 +1,6 @@
+{
+    "INPUT": {
+        "class": "File",
+        "location": "ref.fasta"
+    }
+}

v1.0/v1.0/initialworkdirrequirement-docker-out.cwl

@@ -0,0 +1,30 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+
+requirements:
+  - class: DockerRequirement
+    dockerPull: debian:8
+  - class: InitialWorkDirRequirement
+    listing:
+      - $(inputs.INPUT)
+
+class: CommandLineTool
+
+inputs:
+  - id: INPUT
+    type: File
+
+outputs:
+  - id: OUTPUT
+    type: File
+    outputBinding:
+      glob: $(inputs.INPUT.basename)
+    secondaryFiles:
+      - .fai
+
+arguments:
+  - valueFrom: $(inputs.INPUT.basename).fai
+    position: 0
+
+baseCommand: [touch]

v1.1.0-dev1/CommandLineTool-standalone.yml

@@ -0,0 +1,2 @@
+- $import: Process.yml
+- $import: CommandLineTool.yml