From 75bcc94f6cb46d285d76fdce4f5c8649dcb0cd05 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 20 Oct 2020 16:30:56 +0200 Subject: [PATCH 1/6] Add full workflow pipeline --- .github/workflows/workflow_test.yml | 195 +++++++++++++++++++++++----- 1 file changed, 162 insertions(+), 33 deletions(-) diff --git a/.github/workflows/workflow_test.yml b/.github/workflows/workflow_test.yml index 4d8375216..b73d75d18 100644 --- a/.github/workflows/workflow_test.yml +++ b/.github/workflows/workflow_test.yml @@ -4,14 +4,99 @@ env: GALAXY_REPO: https://github.com/galaxyproject/galaxy GALAXY_RELEASE: release_20.09 jobs: + setup: + # Shared 100% with tool testing setup + name: Setup cache and determine changed repositories + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7] + outputs: + galaxy-commit: ${{ steps.galaxy-commit.outputs.galaxy-commit }} + changed-repositories: ${{ steps.discover-repositories.outputs.changed-repositories }} + steps: + - name: Print github context properties + run: | + echo 'event: ${{ github.event_name }}' + echo 'sha: ${{ github.sha }}' + echo 'ref: ${{ github.ref }}' + echo 'head_ref: ${{ github.head_ref }}' + echo 'base_ref: ${{ github.base_ref }}' + echo 'event.before: ${{ github.event.before }}' + echo 'event.after: ${{ github.event.after }}' + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Determine latest galaxy commit + run: echo "GALAXY_HEAD_SHA=$(git ls-remote ${{ env.GALAXY_REPO }} refs/heads/${{ env.GALAXY_RELEASE }} | cut -f1)" >> $GITHUB_ENV + - name: Set galaxy head commit + id: galaxy-commit + run: echo "::set-output name=galaxy-commit::$GALAXY_HEAD_SHA" + - name: Cache .cache/pip + uses: actions/cache@v2 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA + - name: Cache .planemo + uses: actions/cache@v2 + id: cache-planemo + with: + path: ~/.planemo + key: planemo_cache_py_${{ matrix.python-version }}_gxy_$GALAXY_HEAD_SHA + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: mvdbeek/planemo-discover-action@workflow_enhancements + id: discover-repositories + with: + create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }} + galaxy-branch: ${{ env.GALAXY_RELEASE }} + galaxy-source: ${{ env.GALAXY_REPO }} + + # Planemo lint the changed repositories + lint: + # Only difference with tool linting step is `lint-workflow` + name: Lint tools + needs: setup + if: needs.setup.outputs.changed-repositories != '' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [3.7] + steps: + # checkout the repository + # and use it as the current working directory + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache .cache/pip + uses: actions/cache@v2 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-commit }} + - uses: mvdbeek/planemo-discover-action@workflow_enhancements + id: discover-repositories + with: + changed-repositories: ${{ needs.setup.outputs.changed-repositories }} + lint-workflows: true + test: - name: Setup cache, determine changed workflows and test these workflows + name: Test tools + # This job runs on Linux runs-on: ubuntu-latest + needs: setup + if: needs.setup.outputs.changed-repositories != '' strategy: + fail-fast: false matrix: + chunk: [0, 1, 2, 3] python-version: [3.7] - workflow: - - workflows/sars-cov-2-variant-calling/workflow.ga services: postgres: image: postgres:11 @@ -22,51 +107,88 @@ jobs: ports: - 5432:5432 steps: - - name: Print github context properties - run: | - echo 'event: ${{ github.event_name }}' - echo 'sha: ${{ github.sha }}' - echo 'ref: ${{ github.ref }}' - echo 'head_ref: ${{ github.head_ref }}' - echo 'base_ref: ${{ github.base_ref }}' - echo 'event.before: ${{ github.event.before }}' - echo 'event.after: ${{ github.event.after }}' + # checkout the repository + # and use it as the current working directory - uses: actions/checkout@v2 with: fetch-depth: 1 - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - run: pip install https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip - - name: Run planemo workflow_lint - run: planemo workflow_lint --fail_level error ${{ matrix.workflow }} - name: Cache .cache/pip uses: actions/cache@v2 id: cache-pip with: path: ~/.cache/pip - key: pip_cache_${{ matrix.python-version }}_${{ env.GALAXY_RELEASE }} - - name: Setup cmvfs - run: ./.scripts/setup_cvmfs.sh - - run: planemo test --no_paste_test_data_paths --galaxy_source https://github.com/mvdbeek/galaxy --galaxy_branch "$GALAXY_RELEASE" --database_connection postgresql://postgres:postgres@localhost:5432/galaxy --no_conda_auto_init --no_dependency_resolution --biocontainers --shed_tool_conf /cvmfs/main.galaxyproject.org/config/shed_tool_conf.xml --no_shed_install --tool_data_table /cvmfs/data.galaxyproject.org/byhand/location/tool_data_table_conf.xml --tool_data_table /cvmfs/data.galaxyproject.org/managed/location/tool_data_table_conf.xml --docker_extra_volume /cvmfs ${{ matrix.workflow }} || true - - name: Copy artifacts into place - run: | - mkdir upload - mv tool_test_output.json tool_test_output.html upload/ + key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-commit }} + - name: Cache .planemo + uses: actions/cache@v2 + id: cache-planemo + with: + path: ~/.planemo + key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup.outputs.galaxy-commit }} + - run: ./.scripts/setup_cvmfs.sh + - uses: mvdbeek/planemo-discover-action@workflow_enhancements + with: + changed-repositories: ${{ needs.setup.outputs.changed-repositories }} + test-workflows: true + combine-outputs: true + chunk-count: 4 + chunk: ${{ matrix.chunk}} + - uses: actions/upload-artifact@v2.0.1 + with: + name: 'Workflow test output ${{ matrix.chunk }}' + path: upload + + # - combine the results of the test chunks (which will never fail due + # to `|| true`) and create a global test report as json and html which + # is provided as artifact + # - check if any tool test actually failed (by lookup in the combined json) + # and fail this step if this is the case + combine_outputs: + # same as tool step + name: Combine chunked test results + needs: [setup, test] + strategy: + matrix: + python-version: [3.7] + # This job runs on Linux + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v2 + with: + path: artifacts + - uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Cache .cache/pip + uses: actions/cache@v2 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_py_${{ matrix.python-version }}_gxy_{{ needs.setup.outputs.galaxy-commit }} + - name: Install jq + run: sudo apt-get install jq + - uses: mvdbeek/planemo-discover-action@workflow_enhancements + with: + changed-repositories: ${{ needs.setup.outputs.changed-repositories }} + combine-outputs: true + html-report: true - uses: actions/upload-artifact@v2.0.1 with: - name: 'Workflow test report' + name: 'All tool test results' path: upload - name: Check status of combined status run: | - if jq '.["tests"][]["data"]["status"]' upload/tool_test_output.json | grep -v "success"; then - echo "Unsuccessful tests found, inspect the 'Workflow test report' artifact for details." + if jq '.["tests"][]["data"]["status"]' upload/test_output.json | grep -v "success"; then + echo "Unsuccessful tests found, inspect the 'All workflow test results' artifact for details." exit 1 fi + # deploy workflows to organization deploy: name: Deploy - needs: [test] + needs: [setup,test,combine_outputs] strategy: matrix: python-version: [3.7] @@ -79,9 +201,16 @@ jobs: - uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} - - name: Install planemo - run: pip install https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip - - name: Create github repo - run: planemo workflow_upload ${{ matrix.workflow }} - env: - GITHUB_TOKEN: ${{ secrets.IWC_WORKFLOWS_BOT_TOKEN }} + - name: Cache .cache/pip + uses: actions/cache@v2 + id: cache-pip + with: + path: ~/.cache/pip + key: pip_cache_py_${{ matrix.python-version }}_gxy_{{ needs.setup.outputs.galaxy-commit }} + - name: Update or create repositories in github namespace + uses: mvdbeek/planemo-discover-action@master + with: + changed-repositories: ${{ needs.setup.outputs.changed-repositories }} + workflow-upload: true + workflow-namespace: iwc-workflows + github-token: ${{ secrets.IWC_WORKFLOWS_BOT_TOKEN }} From 9ab2d845c2d1c58b464bf58ddb8b004034d261d6 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 20 Oct 2020 17:57:43 +0200 Subject: [PATCH 2/6] Update workflow version --- workflows/sars-cov-2-variant-calling/CHANGELOG.md | 4 ++++ workflows/sars-cov-2-variant-calling/README.md | 2 ++ workflows/sars-cov-2-variant-calling/workflow.ga | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 workflows/sars-cov-2-variant-calling/README.md diff --git a/workflows/sars-cov-2-variant-calling/CHANGELOG.md b/workflows/sars-cov-2-variant-calling/CHANGELOG.md index 0f520e212..6fc6303ef 100644 --- a/workflows/sars-cov-2-variant-calling/CHANGELOG.md +++ b/workflows/sars-cov-2-variant-calling/CHANGELOG.md @@ -1,3 +1,7 @@ +0.1.4 +----- +Bump workflow version + 0.1.3 ----- Add readme diff --git a/workflows/sars-cov-2-variant-calling/README.md b/workflows/sars-cov-2-variant-calling/README.md new file mode 100644 index 000000000..75f357212 --- /dev/null +++ b/workflows/sars-cov-2-variant-calling/README.md @@ -0,0 +1,2 @@ +This is the main production workflow for calling variants on Illumina data +-------------------------------------------------------------------------- diff --git a/workflows/sars-cov-2-variant-calling/workflow.ga b/workflows/sars-cov-2-variant-calling/workflow.ga index 34ae780a1..21a2a7311 100644 --- a/workflows/sars-cov-2-variant-calling/workflow.ga +++ b/workflows/sars-cov-2-variant-calling/workflow.ga @@ -2,7 +2,7 @@ "a_galaxy_workflow": "true", "annotation": "Downloads reads from SRA and runs SE and PE COVID-19 variation workflows", "format-version": "0.1", - "release": "0.1.3", + "release": "0.1.4", "name": "Download and SE+PE Illumina Covid Variation Workflow", "steps": { "0": { From 8f4a4e2681152b3998ceb7702b72c9708126a0d2 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 20 Oct 2020 19:03:55 +0200 Subject: [PATCH 3/6] Use planemo branch --- .github/workflows/workflow_test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/workflow_test.yml b/.github/workflows/workflow_test.yml index b73d75d18..5d485c720 100644 --- a/.github/workflows/workflow_test.yml +++ b/.github/workflows/workflow_test.yml @@ -50,6 +50,7 @@ jobs: - uses: mvdbeek/planemo-discover-action@workflow_enhancements id: discover-repositories with: + planemo-version: https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }} galaxy-branch: ${{ env.GALAXY_RELEASE }} galaxy-source: ${{ env.GALAXY_REPO }} @@ -83,6 +84,7 @@ jobs: - uses: mvdbeek/planemo-discover-action@workflow_enhancements id: discover-repositories with: + planemo-version: https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip changed-repositories: ${{ needs.setup.outputs.changed-repositories }} lint-workflows: true @@ -130,6 +132,7 @@ jobs: - run: ./.scripts/setup_cvmfs.sh - uses: mvdbeek/planemo-discover-action@workflow_enhancements with: + planemo-version: https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip changed-repositories: ${{ needs.setup.outputs.changed-repositories }} test-workflows: true combine-outputs: true @@ -171,6 +174,7 @@ jobs: run: sudo apt-get install jq - uses: mvdbeek/planemo-discover-action@workflow_enhancements with: + planemo-version: https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip changed-repositories: ${{ needs.setup.outputs.changed-repositories }} combine-outputs: true html-report: true @@ -210,6 +214,7 @@ jobs: - name: Update or create repositories in github namespace uses: mvdbeek/planemo-discover-action@master with: + planemo-version: https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip changed-repositories: ${{ needs.setup.outputs.changed-repositories }} workflow-upload: true workflow-namespace: iwc-workflows From aef0d5123e90255839f0284fc2b11625700c1dff Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 23 Oct 2020 17:20:35 +0200 Subject: [PATCH 4/6] Change something --- workflows/sars-cov-2-variant-calling/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflows/sars-cov-2-variant-calling/README.md b/workflows/sars-cov-2-variant-calling/README.md index 75f357212..f38a9f5a5 100644 --- a/workflows/sars-cov-2-variant-calling/README.md +++ b/workflows/sars-cov-2-variant-calling/README.md @@ -1,2 +1,4 @@ This is the main production workflow for calling variants on Illumina data -------------------------------------------------------------------------- + +Description will be added soon. From d6591a080da54459728e23030380bd9fdf800e28 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 2 Nov 2020 12:46:05 +0100 Subject: [PATCH 5/6] Use planemo branch that supports testing against directory with .dockstore.yml --- .github/workflows/workflow_test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/workflow_test.yml b/.github/workflows/workflow_test.yml index 5d485c720..077c794a6 100644 --- a/.github/workflows/workflow_test.yml +++ b/.github/workflows/workflow_test.yml @@ -50,7 +50,7 @@ jobs: - uses: mvdbeek/planemo-discover-action@workflow_enhancements id: discover-repositories with: - planemo-version: https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip + planemo-version: https://github.com/mvdbeek/planemo/archive/allow_testing_dir_of_workflows.zip create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }} galaxy-branch: ${{ env.GALAXY_RELEASE }} galaxy-source: ${{ env.GALAXY_REPO }} @@ -84,7 +84,7 @@ jobs: - uses: mvdbeek/planemo-discover-action@workflow_enhancements id: discover-repositories with: - planemo-version: https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip + planemo-version: https://github.com/mvdbeek/planemo/archive/allow_testing_dir_of_workflows.zip changed-repositories: ${{ needs.setup.outputs.changed-repositories }} lint-workflows: true @@ -132,7 +132,7 @@ jobs: - run: ./.scripts/setup_cvmfs.sh - uses: mvdbeek/planemo-discover-action@workflow_enhancements with: - planemo-version: https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip + planemo-version: https://github.com/mvdbeek/planemo/archive/allow_testing_dir_of_workflows.zip changed-repositories: ${{ needs.setup.outputs.changed-repositories }} test-workflows: true combine-outputs: true @@ -174,7 +174,7 @@ jobs: run: sudo apt-get install jq - uses: mvdbeek/planemo-discover-action@workflow_enhancements with: - planemo-version: https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip + planemo-version: https://github.com/mvdbeek/planemo/archive/allow_testing_dir_of_workflows.zip changed-repositories: ${{ needs.setup.outputs.changed-repositories }} combine-outputs: true html-report: true @@ -214,7 +214,7 @@ jobs: - name: Update or create repositories in github namespace uses: mvdbeek/planemo-discover-action@master with: - planemo-version: https://github.com/mvdbeek/planemo/archive/ci_find_workflows.zip + planemo-version: https://github.com/mvdbeek/planemo/archive/allow_testing_dir_of_workflows.zip changed-repositories: ${{ needs.setup.outputs.changed-repositories }} workflow-upload: true workflow-namespace: iwc-workflows From 5b409f6b79203d1960955339d8da1568a0227c95 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 2 Nov 2020 17:55:15 +0100 Subject: [PATCH 6/6] Minor tweaks --- .github/workflows/workflow_test.yml | 4 ++-- workflows/sars-cov-2-variant-calling/workflow.ga | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/workflow_test.yml b/.github/workflows/workflow_test.yml index 077c794a6..53d46394d 100644 --- a/.github/workflows/workflow_test.yml +++ b/.github/workflows/workflow_test.yml @@ -58,7 +58,7 @@ jobs: # Planemo lint the changed repositories lint: # Only difference with tool linting step is `lint-workflow` - name: Lint tools + name: Lint workflows needs: setup if: needs.setup.outputs.changed-repositories != '' runs-on: ubuntu-latest @@ -89,7 +89,7 @@ jobs: lint-workflows: true test: - name: Test tools + name: Test workflows # This job runs on Linux runs-on: ubuntu-latest needs: setup diff --git a/workflows/sars-cov-2-variant-calling/workflow.ga b/workflows/sars-cov-2-variant-calling/workflow.ga index 21a2a7311..b43b3e0bf 100644 --- a/workflows/sars-cov-2-variant-calling/workflow.ga +++ b/workflows/sars-cov-2-variant-calling/workflow.ga @@ -2,7 +2,7 @@ "a_galaxy_workflow": "true", "annotation": "Downloads reads from SRA and runs SE and PE COVID-19 variation workflows", "format-version": "0.1", - "release": "0.1.4", + "release": "0.1.5", "name": "Download and SE+PE Illumina Covid Variation Workflow", "steps": { "0": {