Refactor CI workflow and enable parallelization #128
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| test-durations: test-durations-${{ github.ref }} | |
| test-durations-file: .test_durations | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| digest: ${{ steps.build.outputs.digest }} | |
| version: ${{ steps.build.outputs.version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Build Docker image | |
| id: build | |
| uses: ./.github/actions/build-docker | |
| with: | |
| username: ${{ secrets.DOCKER_USER }} | |
| password: ${{ secrets.DOCKER_PASS }} | |
| push: true | |
| test_make_docker_configuration: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v2 | |
| - name: Install dependencies | |
| shell: bash | |
| run: npm ci | |
| - name: Check make/docker configuration | |
| shell: bash | |
| run: | | |
| docker compose version | |
| npm exec jest -- ./tests/make --runInBand | |
| check: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Create failure | |
| id: failure | |
| uses: ./.github/actions/run-docker | |
| with: | |
| digest: ${{ needs.build.outputs.digest }} | |
| version: ${{ needs.build.outputs.version }} | |
| run: | | |
| exit 1 | |
| continue-on-error: true | |
| - name: Verify failure | |
| if: always() | |
| run: | | |
| if [ "${{ steps.failure.outcome }}" -ne "failure" ]; then | |
| echo "Expected failure" | |
| exit 1 | |
| fi | |
| - name: Check (special characters in command) | |
| uses: ./.github/actions/run-docker | |
| with: | |
| digest: ${{ needs.build.outputs.digest }} | |
| version: ${{ needs.build.outputs.version }} | |
| run: | | |
| echo 'this is a question?' | |
| echo 'a * is born' | |
| echo 'wow an array []' | |
| - name: Manage py check | |
| uses: ./.github/actions/run-docker | |
| with: | |
| digest: ${{ needs.build.outputs.digest }} | |
| version: ${{ needs.build.outputs.version }} | |
| run: | | |
| make check | |
| - name: Codestyle | |
| uses: ./.github/actions/run-docker | |
| with: | |
| digest: ${{ needs.build.outputs.digest }} | |
| version: ${{ needs.build.outputs.version }} | |
| run: | | |
| make lint-codestyle | |
| docs: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/configure-pages@v4 | |
| - name: Build Docs | |
| uses: ./.github/actions/run-docker | |
| with: | |
| digest: ${{ needs.build.outputs.digest }} | |
| version: ${{ needs.build.outputs.version }} | |
| compose_file: docker-compose.yml:docker-compose.mount.yml | |
| run: | | |
| make docs | |
| - name: Upload artifact | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: 'docs/_build/html' | |
| # Keep this in sync with `docs.yml: deploy_docs` where we deploy the artifact | |
| name: docs | |
| locales: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # Check out on the fork if applicable | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| repository: ${{github.event.pull_request.head.repo.full_name}} | |
| - name: Extract Locales | |
| uses: ./.github/actions/run-docker | |
| with: | |
| digest: ${{ needs.build.outputs.digest }} | |
| version: ${{ needs.build.outputs.version }} | |
| compose_file: docker-compose.yml:docker-compose.mount.yml | |
| run: make extract_locales | |
| - name: Push Locales (dry-run) | |
| shell: bash | |
| run: | | |
| if [[ ${{ github.event.pull_request.head.repo.fork}} == 'true' ]]; then | |
| echo """ | |
| Github actions are not authorized to push from workflows triggered by forks. | |
| We cannot verify if the l10n extraction push will work or not. | |
| Please submit a PR from the base repository if you are modifying l10n extraction scripts. | |
| """ | |
| exit 0 | |
| fi | |
| make push_locales ARGS="--dry-run" | |
| test_main: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| strategy: | |
| matrix: | |
| group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download test durations | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.test-durations }} | |
| path: ${{ env.test-durations-file }} | |
| continue-on-error: true | |
| - name: Test (test_matrix) | |
| uses: ./.github/actions/run-docker | |
| with: | |
| services: '' | |
| digest: ${{ needs.build.outputs.digest }} | |
| version: ${{ needs.build.outputs.version }} | |
| run: | | |
| pytest --store-durations \ | |
| -n auto \ | |
| -m 'not es_tests and not needs_locales_compilation and not static_assets and not internal_routes_allowed' \ | |
| --splits 10 --group ${{ matrix.group }} \ | |
| -v src/olympia/ | |
| - name: Upload test durations | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ${{ env.test-durations-file }} | |
| name: ${{ env.test-durations }}-${{ matrix.group }} | |
| test_needs_locales_compilation: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Test (test_needs_locales_compilation) | |
| uses: ./.github/actions/run-docker | |
| with: | |
| services: '' | |
| digest: ${{ needs.build.outputs.digest }} | |
| version: ${{ needs.build.outputs.version }} | |
| run: | | |
| make test_needs_locales_compilation | |
| test_static_assets: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Test (test_static_assets) | |
| uses: ./.github/actions/run-docker | |
| with: | |
| services: '' | |
| digest: ${{ needs.build.outputs.digest }} | |
| version: ${{ needs.build.outputs.version }} | |
| # TODO: we should remove this once we | |
| # a) update the asset tests to look in the static-assets folder | |
| # b) copy the static file into the container also. | |
| run: | | |
| make update_assets | |
| make test_static_assets | |
| test_internal_routes_allowed: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Test (test_internal_routes_allowed) | |
| uses: ./.github/actions/run-docker | |
| with: | |
| services: '' | |
| digest: ${{ needs.build.outputs.digest }} | |
| version: ${{ needs.build.outputs.version }} | |
| run: | | |
| make test_internal_routes_allowed | |
| test_es_tests: | |
| runs-on: ubuntu-latest | |
| needs: build | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Test (test_es_tests) | |
| uses: ./.github/actions/run-docker | |
| with: | |
| services: '' | |
| digest: ${{ needs.build.outputs.digest }} | |
| version: ${{ needs.build.outputs.version }} | |
| run: | | |
| make test_es_tests | |
| upload_durations: | |
| needs: test_main | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download durations | |
| uses: actions/download-artifact@v4 | |
| - name: Merge durations | |
| run: | | |
| jq -s add test-durations-*/${{ env.test-durations-file }} > ${{ env.test-durations-file }} | |
| cat ${{ env.test-durations-file }} | |
| - name: Upload merged durations | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ${{ env.test-durations-file }} | |
| name: ${{ env.test-durations }} |