Merge pull request #4 from mvdbeek/test_pr3

Test pr3

.github/workflows/workflow_test.yml

@@ -4,14 +4,101 @@ env:
   GALAXY_REPO: https://github.com/galaxyproject/galaxy
   GALAXY_RELEASE: release_20.09
 jobs:
+  setup:
+    # Shared 100% with tool testing setup
+    name: Setup cache and determine changed repositories
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7]
+    outputs:
+      galaxy-commit: ${{ steps.galaxy-commit.outputs.galaxy-commit }}
+      changed-repositories: ${{ steps.discover-repositories.outputs.changed-repositories }}
+    steps:
+    - name: Print github context properties
+      run: |
+        echo 'event: ${{ github.event_name }}'
+        echo 'sha: ${{ github.sha }}'
+        echo 'ref: ${{ github.ref }}'
+        echo 'head_ref: ${{ github.head_ref }}'
+        echo 'base_ref: ${{ github.base_ref }}'
+        echo 'event.before: ${{ github.event.before }}'
+        echo 'event.after: ${{ github.event.after }}'
+    - uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Determine latest galaxy commit
+      run: echo "GALAXY_HEAD_SHA=$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1)" >> $GITHUB_ENV
+    - name: Set galaxy head commit
+      id: galaxy-commit
+      run: echo "::set-output name=galaxy-commit::$GALAXY_HEAD_SHA"
+    - name: Cache .cache/pip
+      uses: actions/cache@v2
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA
+    - name: Cache .planemo
+      uses: actions/cache@v2
+      id: cache-planemo
+      with:
+        path: ~/.planemo
+        key: planemo_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - uses: mvdbeek/planemo-discover-action@workflow_enhancements
+      id: discover-repositories
+      with:
+        planemo-version: https://github.com/mvdbeek/planemo/archive/allow_testing_dir_of_workflows.zip
+        create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }}
+        galaxy-branch: ${{ env.GALAXY_RELEASE }}
+        galaxy-source: ${{ env.GALAXY_REPO }}
+
+  # Planemo lint the changed repositories
+  lint:
+    # Only difference with tool linting step is `lint-workflow`
+    name: Lint workflows
+    needs: setup
+    if: needs.setup.outputs.changed-repositories != ''
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.7]
+    steps:
+    # checkout the repository
+    # and use it as the current working directory
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
+    - uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Cache .cache/pip
+      uses: actions/cache@v2
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-commit }}
+    - uses: mvdbeek/planemo-discover-action@workflow_enhancements
+      id: discover-repositories
+      with:
+         planemo-version: https://github.com/mvdbeek/planemo/archive/allow_testing_dir_of_workflows.zip
+         changed-repositories: ${{ needs.setup.outputs.changed-repositories }}
+         lint-workflows: true
+
   test:
-    name: Setup cache, determine changed workflows and test these workflows
+    name: Test workflows
+    # This job runs on Linux
     runs-on: ubuntu-latest
+    needs: setup
+    if: needs.setup.outputs.changed-repositories != ''
     strategy:
+      fail-fast: false
       matrix:
+        chunk: [0, 1, 2, 3]
         python-version: [3.7]
-        workflow:
-          - workflows/sars-cov-2-variant-calling/workflow.ga
     services:
       postgres:
         image: postgres:11
@@ -22,51 +109,90 @@ jobs:
         ports:
           - 5432:5432
     steps:
-    - name: Print github context properties
-      run: |
-        echo 'event: ${{ github.event_name }}'
-        echo 'sha: ${{ github.sha }}'
-        echo 'ref: ${{ github.ref }}'
-        echo 'head_ref: ${{ github.head_ref }}'
-        echo 'base_ref: ${{ github.base_ref }}'
-        echo 'event.before: ${{ github.event.before }}'
-        echo 'event.after: ${{ github.event.after }}'
+    # checkout the repository
+    # and use it as the current working directory
     - uses: actions/checkout@v2
       with:
         fetch-depth: 1
     - uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
-    - run: pip install https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip
-    - name: Run planemo workflow_lint
-      run: planemo workflow_lint --fail_level error ${{ matrix.workflow }}
     - name: Cache .cache/pip
       uses: actions/cache@v2
       id: cache-pip
       with:
         path: ~/.cache/pip
-        key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }}
-    - name: Setup cmvfs
-      run: ./.scripts/setup_cvmfs.sh
-    - run: planemo test --no_paste_test_data_paths --galaxy_source https://github.com/mvdbeek/galaxy --galaxy_branch "$GALAXY_RELEASE"  --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --no_conda_auto_init --no_dependency_resolution --biocontainers --shed_tool_conf /cvmfs/main.galaxyproject.org/config/shed_tool_conf.xml --no_shed_install --tool_data_table /cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml --tool_data_table /cvmfs/data.galaxyproject.org/managed/location/tool_data_table_conf.xml --docker_extra_volume /cvmfs ${{ matrix.workflow }} || true
-    - name: Copy artifacts into place
-      run: |
-        mkdir upload
-        mv tool_test_output.json tool_test_output.html upload/
+        key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-commit }}
+    - name: Cache .planemo
+      uses: actions/cache@v2
+      id: cache-planemo
+      with:
+        path: ~/.planemo
+        key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-commit }}
+    - run: ./.scripts/setup_cvmfs.sh
+    - uses: mvdbeek/planemo-discover-action@workflow_enhancements
+      with:
+         planemo-version: https://github.com/mvdbeek/planemo/archive/allow_testing_dir_of_workflows.zip
+         changed-repositories: ${{ needs.setup.outputs.changed-repositories }}
+         test-workflows: true
+         combine-outputs: true
+         chunk-count: 4
+         chunk: ${{ matrix.chunk}}
+    - uses: actions/upload-artifact@v2.0.1
+      with:
+        name: 'Workflow test output ${{ matrix.chunk }}'
+        path: upload
+
+  # - combine the results of the test chunks (which will never fail due
+  #   to `|| true`) and create a global test report as json and html which
+  #   is provided as artifact
+  # - check if any tool test actually failed (by lookup in the combined json)
+  #   and fail this step if this is the case
+  combine_outputs:
+    # same as tool step
+    name: Combine chunked test results
+    needs: [setup, test]
+    strategy:
+      matrix:
+        python-version: [3.7]
+    # This job runs on Linux
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/download-artifact@v2
+      with:
+        path: artifacts
+    - uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Cache .cache/pip
+      uses: actions/cache@v2
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: pip_cache_py_${{ matrix.python-version }}_gxy_{{ needs.setup.outputs.galaxy-commit }}
+    - name: Install jq
+      run: sudo apt-get install jq
+    - uses: mvdbeek/planemo-discover-action@workflow_enhancements
+      with:
+         planemo-version: https://github.com/mvdbeek/planemo/archive/allow_testing_dir_of_workflows.zip
+         changed-repositories: ${{ needs.setup.outputs.changed-repositories }}
+         combine-outputs: true
+         html-report: true
     - uses: actions/upload-artifact@v2.0.1
       with:
-        name: 'Workflow test report'
+        name: 'All tool test results'
         path: upload
     - name: Check status of combined status
       run: |
-        if jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success"; then
-            echo "Unsuccessful tests found, inspect the 'Workflow test report' artifact for details."
+        if jq '.["tests"][]["data"]["status"]' upload/test_output.json | grep -v "success"; then
+            echo "Unsuccessful tests found, inspect the 'All workflow test results' artifact for details."
             exit 1
         fi
+
   # deploy workflows to organization
   deploy:
     name: Deploy
-    needs: [test]
+    needs: [setup,test,combine_outputs]
     strategy:
       matrix:
         python-version: [3.7]
@@ -79,9 +205,17 @@ jobs:
     - uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install planemo
-      run: pip install https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip
-    - name: Create github repo
-      run: planemo workflow_upload ${{ matrix.workflow }}
-      env:
-        GITHUB_TOKEN: ${{ secrets.IWC_WORKFLOWS_BOT_TOKEN }}
+    - name: Cache .cache/pip
+      uses: actions/cache@v2
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: pip_cache_py_${{ matrix.python-version }}_gxy_{{ needs.setup.outputs.galaxy-commit }}
+    - name: Update or create repositories in github namespace
+      uses: mvdbeek/planemo-discover-action@master
+      with:
+         planemo-version: https://github.com/mvdbeek/planemo/archive/allow_testing_dir_of_workflows.zip
+         changed-repositories: ${{ needs.setup.outputs.changed-repositories }}
+         workflow-upload: true
+         workflow-namespace: iwc-workflows
+         github-token: ${{ secrets.IWC_WORKFLOWS_BOT_TOKEN }}

workflows/sars-cov-2-variant-calling/CHANGELOG.md

@@ -1,6 +1,10 @@
+0.1.5
+-----
+And again
+
 0.1.4
 -----
-Bump version
+Bump workflow version
 
 0.1.3
 -----

workflows/sars-cov-2-variant-calling/README.md

@@ -0,0 +1,4 @@
+This is the main production workflow for calling variants on Illumina data
+--------------------------------------------------------------------------
+
+Description will be added soon.

workflows/sars-cov-2-variant-calling/workflow.ga

@@ -2,7 +2,7 @@
   "a_galaxy_workflow": "true",
   "annotation": "Downloads reads from SRA and runs SE and PE COVID-19 variation workflows",
   "format-version": "0.1",
-  "release": "0.1.4",
+  "release": "0.1.5",
   "name": "Download and SE+PE Illumina Covid Variation Workflow",
   "steps": {
     "0": {