diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2d4bf65..3da7191 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,11 +26,7 @@ jobs: test-pathogen-repo-ci: uses: ./.github/workflows/pathogen-repo-ci.yaml with: - # XXX TODO: Revert back to nextstrain/zika (for consistency with - # elsewhere) once pathogen-repo-ci has support for running workflows in - # subdirs (i.e. for our new repo layout). - # -trs, 30 Jan 2024 - repo: nextstrain/ebola + repo: nextstrain/zika artifact-name: outputs-test-pathogen-repo-ci secrets: inherit diff --git a/.github/workflows/pathogen-repo-ci.yaml b/.github/workflows/pathogen-repo-ci.yaml index d4a9f9e..71149f9 100644 --- a/.github/workflows/pathogen-repo-ci.yaml +++ b/.github/workflows/pathogen-repo-ci.yaml @@ -3,6 +3,15 @@ # workflow) in this repo for an example of what the caller workflow looks like. name: CI +defaults: + run: + # This is the same as GitHub Action's `bash` keyword as of 20 June 2023: + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell + # + # Completely spelling it out here so that GitHub can't change it out from under us + # and we don't have to refer to the docs to know the expected behavior. + shell: bash --noprofile --norc -eo pipefail {0} + on: workflow_call: inputs: @@ -65,15 +74,20 @@ on: artifact-name: description: >- - Name to use for build results artifact uploaded at the end of the - workflow. This name will be suffixed with other information from the - workflow job matrix to distinguish each artifact in a workflow run. + A base name to use for the uploaded artifacts from the + build. This will be concatenated with the directory name + used to invoke the build to generate the full artifact file + name. + + Defaults to `ci-outputs`. - If you're invoking this workflow multiple times from the same calling - workflow, you should set this. Otherwise, the default of "outputs" - is probably fine. + N.b., you almost certainly don't need to change this, unless + you're running multiple invocations of this action in a + single workflow. Then you should provide unique values to + ensure you can access the artifacts from all the builds + you're doing. type: string - default: outputs + default: ci-outputs required: false continue-on-error: @@ -85,8 +99,12 @@ on: permissions: contents: read + id-token: write packages: read +env: + NEXTSTRAIN_GITHUB_DIR: .git/nextstrain/.github + jobs: configuration: runs-on: ubuntu-latest @@ -101,8 +119,17 @@ jobs: outputs: runtimes: ${{ steps.inputs.outputs.runtimes }} - build: - needs: configuration + workflow-context: + runs-on: ubuntu-latest + steps: + - id: workflow-context + uses: nextstrain/.github/actions/workflow-context@master + outputs: + repository: ${{ steps.workflow-context.outputs.repository }} + sha: ${{ steps.workflow-context.outputs.sha }} + + nextstrain-build: + needs: [configuration, workflow-context] strategy: fail-fast: false matrix: @@ -196,58 +223,44 @@ jobs: ' | tee -a "$GITHUB_ENV" - - uses: actions/checkout@v4 + - name: Checkout ${{ inputs.repo }} + uses: actions/checkout@v4 with: repository: ${{ inputs.repo }} - # XXX TODO: It would be better for this to call setup-nextstrain-cli - # using the same ref that this workflow was called with (e.g. if this - # workflow was invoked by the caller workflow with @foo than we invoke - # the action with @foo too), but it's not currently possible to figure - # out that ref. See discussion around this (including results of some - # investigation I did): - # - # - https://github.community/t/reusable-workflows-get-the-ref-inside-the-called-workflow/224109 - # - https://github.community/t/ref-head-in-reusable-workflows/203690/92 - # - # Once we can figure out that ref, then we can actions/checkout our - # nextstrain/.github repo at that ref as a sidecar path somewhere and - # then invoke the setup-nextstrain-cli action using a local file path - # instead of a remote owner/repo path. This separate checkout will be - # necessary since the "uses:" key can't be interpolated (${{…}}) with - # context vars. - # - # For now, update the hardcoded ref (e.g. @90af34…) below when you make - # future changes to setup-nextstrain-cli. - # - # [ Update 16 Feb 2024: We solved this for pathogen-repo-build.yaml, but - # because it required a new permission on the GitHub tokens (id-token: - # write) we decided not to update this workflow (yet?) to use the same - # approach. -trs ] - # - # -trs, 28 April 2022 - - uses: nextstrain/.github/actions/setup-nextstrain-cli@c1191de9d5e1a30e91d70b0fd1041d97ed1b2496 + # Need to run this after the build repo is cloned so that cloning the + # build repo does not overwrite the .git dir and remove the extra support files + # that we need from nextstrain/.github repo + - name: Checkout ${{ needs.workflow-context.outputs.repository }} (sha ${{ needs.workflow-context.outputs.sha }}) + uses: actions/checkout@v4 with: + repository: ${{ needs.workflow-context.outputs.repository }} + ref: ${{ needs.workflow-context.outputs.sha }} + path: ${{ env.NEXTSTRAIN_GITHUB_DIR }} + + - name: Set up Nextstrain runtime ${{ matrix.runtime }} + uses: ./.git/nextstrain/.github/actions/setup-nextstrain-cli + with: + cli-version: ">=8.3.0" runtime: ${{ matrix.runtime }} - - name: Copy example data - run: | - if [[ -d example_data ]]; then - mkdir -p data/ - cp -r -v example_data/* data/ - else - echo No example data to copy. - fi + - name: Run ingest + uses: ./.git/nextstrain/.github/actions/run-nextstrain-ci-build + with: + directory: ingest + runtime: ${{ matrix.runtime }} + artifact-name: ${{ inputs.artifact-name }} - - run: nextstrain build . ${{ inputs.build-args }} + - name: Run phylogenetic + uses: ./.git/nextstrain/.github/actions/run-nextstrain-ci-build + with: + directory: phylogenetic + runtime: ${{ matrix.runtime }} + artifact-name: ${{ inputs.artifact-name }} - - if: always() - uses: actions/upload-artifact@v4 + - name: Run nextclade + uses: ./.git/nextstrain/.github/actions/run-nextstrain-ci-build with: - name: ${{ inputs.artifact-name }}-${{ matrix.runtime }} - path: | - auspice/ - results/ - benchmarks/ - logs/ - .snakemake/log/ + directory: nextclade + runtime: ${{ matrix.runtime }} + artifact-name: ${{ inputs.artifact-name }} diff --git a/README.md b/README.md index 261e48e..b9a506a 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ See also GitHub's [documentation on issue and PR templates](https://docs.github. Invoked by our GitHub Actions workflows, including the reusable workflows below. +- [Run Nextstrain CI Build](actions/run-nextstrain-ci-build/action.yaml) - [Setup Nextstrain CLI](actions/setup-nextstrain-cli/action.yaml) - [shellcheck](actions/shellcheck/action.yaml) - [Setup SSH](actions/setup-ssh/action.yaml) access to runner machine diff --git a/actions/run-nextstrain-ci-build/action.yaml b/actions/run-nextstrain-ci-build/action.yaml new file mode 100644 index 0000000..66cf141 --- /dev/null +++ b/actions/run-nextstrain-ci-build/action.yaml @@ -0,0 +1,58 @@ +name: run-nextstrain-ci-build +description: >- + Runs a single `nextstrain build` command in a given sub-directory of + a pathogen repo. Must be provided with the name of the sub-directory + and the runtime to use. Requires that the Nextstrain CLI runtime + already be provisioned (e.g., via the `setup-nextstrain-cli` action + in this repo). + + Note that this action exists primarily as a means to keep the + `pathogen-repo-ci` workflow DRY; it is unlikely to be useful outside + the context of that specific workflow. + +inputs: + artifact-name: + description: >- + Name to append to the build directory to generate a + unique artifact name for the upload action. + type: string + required: true + directory: + description: The name of the sub-directory to run the build from + type: string + required: true + runtime: + description: Nextstrain runtime to use for the build + type: string + required: true + +runs: + using: "composite" + steps: + - id: run-build + env: + DIR: ${{ inputs.directory }} + run: | + if [[ -f nextstrain-pathogen.yaml && -f "$DIR"/Snakefile && -f "$DIR"/build-configs/ci/config.yaml ]]; then + nextstrain check-setup ${{ inputs.runtime }} --set-default + nextstrain build "$DIR" --configfile build-configs/ci/config.yaml + else + echo "Skipping $DIR build due to one or more missing files." + for i in nextstrain-pathogen.yaml "$DIR"/Snakefile "$DIR"/build-configs/ci/config.yaml; do + [[ -f $i ]] || echo missing $i + done + fi + shell: bash + + - id: upload-artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }}-${{ inputs.directory }}-${{ inputs.runtime }} + if-no-files-found: ignore + path: | + ${{ inputs.directory }}/.snakemake/log/ + ${{ inputs.directory }}/auspice/ + ${{ inputs.directory }}/benchmarks/ + ${{ inputs.directory }}/logs/ + ${{ inputs.directory }}/results/ diff --git a/actions/setup-nextstrain-cli/action.yaml b/actions/setup-nextstrain-cli/action.yaml index ace540a..be6db12 100644 --- a/actions/setup-nextstrain-cli/action.yaml +++ b/actions/setup-nextstrain-cli/action.yaml @@ -1,7 +1,3 @@ -# If you update this action, please update the hardcoded ref to it in -# .github/workflows/pathogen-repo-ci.yaml. See the commentary there for why -# it's necessary. - name: Setup Nextstrain CLI description: >- This GitHub Actions action is intended to be called by workflows in our other