Tests #20
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| --- | |
| name: Tests | |
| on: # yamllint disable-line rule:truthy | |
| schedule: | |
| - cron: '28 0 * * *' | |
| push: | |
| branches: ['main', 'v[0-9]+-[0-9]+-test'] | |
| pull_request: | |
| branches: ['main'] | |
| permissions: | |
| # All other permissions are set to none | |
| contents: read | |
| # Technically read access while waiting for images should be more than enough. However, | |
| # there is a bug in GitHub Actions/Packages and in case private repositories are used, you get a permission | |
| # denied error when attempting to just pull private image, changing the token permission to write solves the | |
| # issue. This is not dangerous, because if it is for "apache/airflow", only maintainers can push ci.yml | |
| # changes. If it is for a fork, then the token is read-only anyway. | |
| packages: write | |
| env: | |
| ANSWER: "yes" | |
| DB_RESET: "true" | |
| VERBOSE: "true" | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_USERNAME: ${{ github.actor }} | |
| # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your repo but by default the | |
| # Airflow one is going to be used | |
| CONSTRAINTS_GITHUB_REPOSITORY: >- | |
| ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' && | |
| secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }} | |
| # In builds from forks, this token is read-only. For scheduled/direct push it is WRITE one | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" | |
| USE_SUDO: "true" | |
| INCLUDE_NOT_READY_PROVIDERS: "true" | |
| AIRFLOW_ENABLE_AIP_44: "true" | |
| MOUNT_SOURCES: "skip" | |
| concurrency: | |
| group: ci-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-info: | |
| name: "Build info" | |
| # The runs-on cannot refer to env. or secrets. context, so we have no | |
| # option but to specify a hard-coded list here. This is "safe", the | |
| # runner checks if the user is an owner or collaborator of the repo | |
| # before running the workflow. | |
| runs-on: ["ubuntu-22.04"] | |
| env: | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| outputs: | |
| # Version of Python used for reproducibility of the packages built | |
| # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing | |
| # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible | |
| # TODO: remove me when we switch to Python 3.9 as minimal version | |
| breeze-python-version: "3.9" | |
| image-tag: ${{ github.event.pull_request.head.sha || github.sha }} | |
| cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} | |
| affected-providers-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.affected-providers-list-as-string }} | |
| upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} | |
| python-versions: ${{ steps.selective-checks.outputs.python-versions }} | |
| python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} | |
| all-python-versions-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} | |
| default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} | |
| kubernetes-versions-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} | |
| kubernetes-combos-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} | |
| default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} | |
| postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} | |
| default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} | |
| mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} | |
| default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} | |
| default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} | |
| default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} | |
| full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} | |
| parallel-test-types-list-as-string: >- | |
| ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }} | |
| include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} | |
| postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} | |
| mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} | |
| sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} | |
| skip-provider-tests: ${{ steps.selective-checks.outputs.skip-provider-tests }} | |
| run-tests: ${{ steps.selective-checks.outputs.run-tests }} | |
| run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} | |
| run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} | |
| run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} | |
| basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} | |
| ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} | |
| prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} | |
| docs-build: ${{ steps.selective-checks.outputs.docs-build }} | |
| mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }} | |
| needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }} | |
| needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} | |
| needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} | |
| needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} | |
| default-branch: ${{ steps.selective-checks.outputs.default-branch }} | |
| default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} | |
| docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }} | |
| skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} | |
| providers-compatibility-checks: ${{ steps.selective-checks.outputs.providers-compatibility-checks }} | |
| helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} | |
| debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} | |
| runs-on: ${{steps.selective-checks.outputs.runs-on}} | |
| is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }} | |
| is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} | |
| is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} | |
| is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} | |
| is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} | |
| is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} | |
| latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} | |
| chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} | |
| has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} | |
| source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} | |
| pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} | |
| in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }} | |
| build-job-description: ${{ steps.source-run-info.outputs.build-job-description }} | |
| canary-run: ${{ steps.source-run-info.outputs.canary-run }} | |
| run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: Fetch incoming commit ${{ github.sha }} with its parent | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.sha }} | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Get information about the Workflow" | |
| id: source-run-info | |
| run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} | |
| - name: Selective checks | |
| id: selective-checks | |
| env: | |
| PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" | |
| COMMIT_REF: "${{ github.sha }}" | |
| VERBOSE: "false" | |
| run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} | |
| - name: env | |
| run: printenv | |
| env: | |
| PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| # Push early BuildX cache to GitHub Registry in Apache repository, This cache does not wait for all the | |
| # tests to complete - it is run very early in the build process for "main" merges in order to refresh | |
| # cache using the current constraints. This will speed up cache refresh in cases when pyproject.toml | |
| # changes or in case of Dockerfile changes. Failure in this step is not a problem (at most it will | |
| # delay cache refresh. It does not attempt to upgrade to newer dependencies. | |
| # We only push CI cache as PROD cache usually does not gain as much from fresh cache because | |
| # it uses prepared airflow and provider packages that invalidate the cache anyway most of the time | |
| push-early-buildx-cache-to-github-registry: | |
| name: Push Early Image Cache | |
| needs: [build-info, build-ci-images] | |
| uses: ./.github/workflows/push-image-cache.yml | |
| permissions: | |
| contents: read | |
| # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
| # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
| # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the | |
| # "in-workflow-build" condition | |
| packages: write | |
| secrets: inherit | |
| with: | |
| cache-type: "Early" | |
| include-prod-images: "false" | |
| push-latest-images: "false" | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| use-uv: "true" | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| # Check that after earlier cache push, breeze command will build quickly | |
| check-that-image-builds-quickly: | |
| timeout-minutes: 5 | |
| name: "Check that image builds quickly" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: | |
| - build-info | |
| - push-early-buildx-cache-to-github-registry | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| UPGRADE_TO_NEWER_DEPENDENCIES: false | |
| PLATFORM: "linux/amd64" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| PYTHON_VERSION: ${{needs.build-info.outputs.default-python-version}} | |
| if: > | |
| needs.build-info.outputs.canary-run == 'true' | |
| && needs.build-info.outputs.default-branch == 'main' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: "Check that image builds quickly" | |
| run: breeze shell --max-time 120 | |
| basic-tests: | |
| name: "Basic tests" | |
| needs: [build-info] | |
| uses: ./.github/workflows/basic-tests.yml | |
| with: | |
| run-www-tests: ${{needs.build-info.outputs.run-www-tests}} | |
| needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} | |
| default-python-version: ${{needs.build-info.outputs.default-python-version}} | |
| breeze-python-version: ${{needs.build-info.outputs.breeze-python-version}} | |
| basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}} | |
| skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}} | |
| canary-run: ${{needs.build-info.outputs.canary-run}} | |
| latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} | |
| build-ci-images: | |
| name: > | |
| ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }} | |
| CI images in-workflow | |
| needs: [build-info] | |
| uses: ./.github/workflows/ci-image-build.yml | |
| permissions: | |
| contents: read | |
| # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
| # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
| # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the | |
| # "in-workflow-build" condition | |
| packages: write | |
| secrets: inherit | |
| with: | |
| do-build: ${{ needs.build-info.outputs.in-workflow-build }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| upload-constraints: "true" | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| use-uv: "true" | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| wait-for-ci-images: | |
| timeout-minutes: 120 | |
| name: "Wait for CI images" | |
| runs-on: ["ubuntu-22.04"] | |
| needs: [build-info, build-ci-images] | |
| if: needs.build-info.outputs.ci-image-build == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| # Force more parallelism for pull even on public images | |
| PARALLELISM: 6 | |
| INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| id: wait-for-images | |
| run: breeze ci-image pull --run-in-parallel --wait-for-image --tag-as-latest | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| generate-constraints: | |
| permissions: | |
| contents: read | |
| timeout-minutes: 70 | |
| name: > | |
| Generate constraints with CI image verifying | |
| ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| VERSION_SUFFIX_FOR_PYPI: "dev0" | |
| INCLUDE_SUCCESS_OUTPUTS: "true" | |
| if: needs.build-info.outputs.ci-image-build == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| with: | |
| python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| run: breeze ci-image pull --run-in-parallel --tag-as-latest | |
| - name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| run: breeze ci-image verify --run-in-parallel | |
| env: | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| - name: "Source constraints" | |
| shell: bash | |
| run: > | |
| breeze release-management generate-constraints --run-in-parallel | |
| --airflow-constraints-mode constraints-source-providers | |
| - name: "No providers constraints" | |
| shell: bash | |
| timeout-minutes: 25 | |
| run: > | |
| breeze release-management generate-constraints --run-in-parallel | |
| --airflow-constraints-mode constraints-no-providers | |
| - name: "Prepare chicken-eggs provider packages" | |
| # In case of provider packages which use latest dev0 version of providers, we should prepare them | |
| # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency | |
| # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 | |
| shell: bash | |
| run: > | |
| breeze release-management prepare-provider-packages --include-not-ready-providers | |
| --package-format wheel --version-suffix-for-pypi dev0 | |
| ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| if: needs.build-info.outputs.chicken-egg-providers != '' | |
| - name: "PyPI constraints" | |
| shell: bash | |
| timeout-minutes: 25 | |
| run: > | |
| breeze release-management generate-constraints --run-in-parallel | |
| --airflow-constraints-mode constraints | |
| --chicken-egg-providers "${{ needs.build-info.outputs.chicken-egg-providers }}" | |
| - name: "Dependency upgrade summary" | |
| shell: bash | |
| run: | | |
| for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do | |
| echo "Summarizing Python $PYTHON_VERSION" | |
| cat "files/constraints-${PYTHON_VERSION}"/*.md >> $GITHUB_STEP_SUMMARY || true | |
| done | |
| - name: "Upload constraint artifacts" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: constraints | |
| path: ./files/constraints-*/constraints-*.txt | |
| retention-days: 7 | |
| if-no-files-found: error | |
| static-checks: | |
| timeout-minutes: 45 | |
| name: "Static checks" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" | |
| if: > | |
| needs.build-info.outputs.basic-checks-only == 'false' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| id: breeze | |
| - name: Cache pre-commit envs | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pre-commit | |
| # yamllint disable-line rule:line-length | |
| key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}" | |
| restore-keys: | | |
| pre-commit-${{steps.breeze.outputs.host-python-version}}- | |
| - name: "Static checks" | |
| run: breeze static-checks --all-files --show-diff-on-failure --color always --initialize-environment | |
| env: | |
| VERBOSE: "false" | |
| SKIP: ${{ needs.build-info.outputs.skip-pre-commits }} | |
| COLUMNS: "250" | |
| SKIP_GROUP_OUTPUT: "true" | |
| DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} | |
| RUFF_FORMAT: "github" | |
| mypy: | |
| name: "MyPy tests" | |
| uses: ./.github/workflows/mypy.yml | |
| needs: [build-info, wait-for-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| secrets: inherit | |
| with: | |
| runs-on: ${{ needs.build-info.outputs.runs-on }} | |
| mypy-folders: ${{ needs.build-info.outputs.mypy-folders }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
| needs-mypy: ${{ needs.build-info.outputs.needs-mypy }} | |
| if: needs.build-info.outputs.latest-versions-only != 'true' | |
| build-docs: | |
| timeout-minutes: 60 | |
| name: "Build docs" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| if: > | |
| needs.build-info.outputs.docs-build == 'true' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - uses: actions/cache@v4 | |
| id: cache-doc-inventories | |
| with: | |
| path: ./docs/_inventory_cache/ | |
| key: docs-inventory-${{ hashFiles('pyproject.toml;') }} | |
| restore-keys: | | |
| docs-inventory-${{ hashFiles('pyproject.toml;') }} | |
| docs-inventory- | |
| - name: "Build docs" | |
| run: > | |
| breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string }} --docs-only | |
| - name: "Clone airflow-site" | |
| run: > | |
| git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site && | |
| echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV" | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Publish docs" | |
| run: > | |
| breeze release-management publish-docs | |
| --override-versioned --run-in-parallel | |
| ${{ needs.build-info.outputs.docs-list-as-string }} | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Generate back references for providers" | |
| run: breeze release-management add-back-references all-providers | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Generate back references for apache-airflow" | |
| run: breeze release-management add-back-references apache-airflow | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Generate back references for docker-stack" | |
| run: breeze release-management add-back-references docker-stack | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Generate back references for helm-chart" | |
| run: breeze release-management add-back-references helm-chart | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| with: | |
| aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} | |
| aws-region: eu-central-1 | |
| - name: "Upload documentation to AWS S3" | |
| if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' | |
| run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs | |
| spellcheck-docs: | |
| timeout-minutes: 120 | |
| name: "Spellcheck docs" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| if: > | |
| needs.build-info.outputs.docs-build == 'true' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - uses: actions/cache@v4 | |
| id: cache-doc-inventories | |
| with: | |
| path: ./docs/_inventory_cache/ | |
| key: docs-inventory-${{ hashFiles('pyproject.toml;') }} | |
| restore-keys: | | |
| docs-inventory-${{ hashFiles('pyproject.toml;') }} | |
| docs-inventory- | |
| - name: "Spellcheck docs" | |
| run: > | |
| breeze build-docs ${{ needs.build-info.outputs.docs-list-as-string }} --spellcheck-only | |
| providers: | |
| name: "Provider checks" | |
| uses: ./.github/workflows/check-providers.yml | |
| needs: [build-info, wait-for-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| secrets: inherit | |
| if: > | |
| needs.build-info.outputs.skip-providers-tests != 'true' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| with: | |
| runs-on: ${{ needs.build-info.outputs.runs-on }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| affected-providers-list-as-string: ${{ needs.build-info.outputs.affected-providers-list-as-string }} | |
| providers-compatibility-checks: ${{ needs.build-info.outputs.providers-compatibility-checks }} | |
| skip-provider-tests: ${{ needs.build-info.outputs.skip-provider-tests }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| test-airflow-release-commands: | |
| timeout-minutes: 80 | |
| name: "Test Airflow release commands" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| if: > | |
| needs.build-info.outputs.is-self-hosted-runner == 'true' | |
| && needs.build-info.outputs.is-airflow-runner == 'false' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: > | |
| Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Cleanup dist files" | |
| run: rm -fv ./dist/* | |
| - name: "Install required packages" | |
| run: python -m pip install --editable ./dev/breeze/ | |
| - name: "Check Airflow create minor branch command" | |
| run: breeze release-management create-minor-branch --version-branch 2-4 -a y | |
| - name: "Check Airflow RC process command" | |
| run: | | |
| breeze release-management start-rc-process --version 2.4.3rc1 --previous-version 2.4.2 -a y | |
| - name: "Check Airflow release process command" | |
| run: | | |
| breeze release-management start-release --release-candidate 2.4.3rc1 --previous-release 2.4.2 -a y | |
| tests-helm: | |
| name: "Helm tests" | |
| uses: ./.github/workflows/helm-tests.yml | |
| needs: [build-info, wait-for-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| secrets: inherit | |
| with: | |
| runs-on: ${{ needs.build-info.outputs.runs-on }} | |
| helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
| if: > | |
| needs.build-info.outputs.needs-helm-tests == 'true' && | |
| needs.build-info.outputs.default-branch == 'main' && | |
| needs.build-info.outputs.latest-versions-only != 'true' | |
| tests-postgres: | |
| name: "Postgres tests" | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| needs: [build-info, wait-for-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| secrets: inherit | |
| with: | |
| runs-on: ${{ needs.build-info.outputs.runs-on }} | |
| backend: "postgres" | |
| test-name: "Postgres" | |
| test-scope: "DB" | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| backend-versions: ${{ needs.build-info.outputs.postgres-versions }} | |
| excludes: ${{ needs.build-info.outputs.postgres-exclude }} | |
| parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-migration-tests: "true" | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| tests-mysql: | |
| name: "MySQL tests" | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| needs: [build-info, wait-for-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| secrets: inherit | |
| with: | |
| runs-on: ${{ needs.build-info.outputs.runs-on }} | |
| backend: "mysql" | |
| test-name: "MySQL" | |
| test-scope: "DB" | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| backend-versions: ${{ needs.build-info.outputs.mysql-versions }} | |
| excludes: ${{ needs.build-info.outputs.mysql-exclude }} | |
| parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| run-migration-tests: "true" | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| tests-sqlite: | |
| name: "Sqlite tests" | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| needs: [build-info, wait-for-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| secrets: inherit | |
| with: | |
| runs-on: ${{ needs.build-info.outputs.runs-on }} | |
| backend: "sqlite" | |
| test-name: "Sqlite" | |
| test-name-separator: "" | |
| test-scope: "DB" | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| # No versions for sqlite | |
| backend-versions: "['']" | |
| excludes: ${{ needs.build-info.outputs.sqlite-exclude }} | |
| parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| run-migration-tests: "true" | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| tests-non-db: | |
| name: "Non-DB tests" | |
| uses: ./.github/workflows/run-unit-tests.yml | |
| needs: [build-info, wait-for-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| secrets: inherit | |
| with: | |
| runs-on: ${{ needs.build-info.outputs.runs-on }} | |
| backend: "sqlite" | |
| test-name: "" | |
| test-name-separator: "" | |
| test-scope: "Non-DB" | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| # No versions for non-db | |
| backend-versions: "['']" | |
| excludes: ${{ needs.build-info.outputs.sqlite-exclude }} | |
| parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| tests-special: | |
| name: "Special tests" | |
| uses: ./.github/workflows/special-tests.yml | |
| needs: [build-info, wait-for-ci-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| secrets: inherit | |
| with: | |
| runs-on: ${{ needs.build-info.outputs.runs-on }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| parallel-test-types-list-as-string: ${{ needs.build-info.outputs.parallel-test-types-list-as-string }} | |
| run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
| default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
| default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| tests-integration-postgres: | |
| timeout-minutes: 130 | |
| name: Integration Tests Postgres | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "postgres" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" | |
| JOB_ID: "integration-postgres" | |
| SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| - name: "Integration Tests Postgres: cassandra" | |
| run: | | |
| breeze testing integration-tests --integration cassandra | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: mongo" | |
| run: | | |
| breeze testing integration-tests --integration mongo | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: pinot" | |
| run: | | |
| breeze testing integration-tests --integration pinot | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: celery" | |
| run: | | |
| breeze testing integration-tests --integration celery | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: trino, kerberos" | |
| run: | | |
| breeze testing integration-tests --integration trino --integration kerberos | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: Kafka" | |
| run: | | |
| breeze testing integration-tests --integration kafka | |
| breeze down | |
| if: needs.build-info.outputs.is-airflow-runner != 'true' | |
| - name: "Integration Tests Postgres: Qdrant" | |
| run: breeze testing integration-tests --integration qdrant | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: "Integration Tests Postgres: all-testable" | |
| run: breeze testing integration-tests --integration all-testable | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: > | |
| Post Tests success: Integration Postgres" | |
| uses: ./.github/actions/post_tests_success | |
| if: success() | |
| - name: > | |
| Post Tests failure: Integration Postgres" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| tests-integration-mysql: | |
| timeout-minutes: 130 | |
| name: Integration Tests MySQL | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-ci-images] | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" | |
| PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" | |
| FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" | |
| DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" | |
| BACKEND: "mysql" | |
| ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| BACKEND_VERSION: "${{needs.build-info.outputs.default-mysql-version}}" | |
| JOB_ID: "integration-mysql" | |
| SKIP_PROVIDER_TESTS: "${{needs.build-info.outputs.skip-provider-tests}}" | |
| if: needs.build-info.outputs.run-tests == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: "Prepare breeze & CI image: ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" | |
| uses: ./.github/actions/prepare_breeze_and_image | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: "Integration Tests MySQL: all-testable" | |
| run: breeze testing integration-tests --integration all-testable | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: > | |
| Post Tests success: Integration MySQL" | |
| uses: ./.github/actions/post_tests_success | |
| if: needs.build-info.outputs.is-airflow-runner == 'true' | |
| - name: > | |
| Post Tests failure: Integration MySQL" | |
| uses: ./.github/actions/post_tests_failure | |
| if: failure() | |
| summarize-warnings: | |
| timeout-minutes: 15 | |
| name: "Summarize warnings" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: | |
| - build-info | |
| - tests-postgres | |
| - tests-sqlite | |
| - tests-mysql | |
| - tests-non-db | |
| - tests-special | |
| - tests-integration-postgres | |
| - tests-integration-mysql | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Download all artifacts from the current build" | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ./artifacts | |
| - name: "Summarize all warnings" | |
| run: | | |
| ls -R ./artifacts/ | |
| cat ./artifacts/test-warnings*/* | sort | uniq | |
| echo | |
| echo Total number of unique warnings $(cat ./artifacts/test-warnings*/* | sort | uniq | wc -l) | |
| build-prod-images: | |
| name: > | |
| ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }} | |
| PROD images in-workflow | |
| needs: [build-info, build-ci-images] | |
| uses: ./.github/workflows/prod-image-build.yml | |
| permissions: | |
| contents: read | |
| # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
| # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
| # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the | |
| # "in-workflow-build" condition | |
| packages: write | |
| secrets: inherit | |
| with: | |
| build-type: "Regular" | |
| do-build: ${{ needs.build-info.outputs.in-workflow-build }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| push-image: "true" | |
| use-uv: "true" | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| prod-image-extra-checks-main: | |
| name: PROD image extra checks (main) | |
| # Here we just need to wait for CI images: we build provider packages and can use source constraints | |
| needs: [build-info, build-ci-images] | |
| uses: ./.github/workflows/prod-image-extra-checks.yml | |
| with: | |
| python-versions: "[ '${{ needs.build-info.outputs.default-python-version }}' ]" | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| if: needs.build-info.outputs.default-branch == 'main' && needs.build-info.outputs.canary-run == 'true' | |
| prod-image-extra-checks-release-branch: | |
| name: PROD image extra checks (release) | |
| # Here we need to wait for generate-constraints to complete because we have to use PyPI constraints | |
| needs: [build-info, generate-constraints] | |
| uses: ./.github/workflows/prod-image-extra-checks.yml | |
| with: | |
| python-versions: "[ '${{ needs.build-info.outputs.default-python-version }}' ]" | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| if: needs.build-info.outputs.default-branch != 'main' && needs.build-info.outputs.canary-run == 'true' | |
| wait-for-prod-images: | |
| timeout-minutes: 80 | |
| name: "Wait for PROD images" | |
| runs-on: ["ubuntu-22.04"] | |
| needs: [build-info, wait-for-ci-images, build-prod-images] | |
| if: needs.build-info.outputs.prod-image-build == 'true' | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| BACKEND: sqlite | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| # Force more parallelism for pull on public images | |
| PARALLELISM: 6 | |
| INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| # We wait for the images to be available either from "build-images.yml' run as pull_request_target | |
| # or from build-prod-images (or build-prod-images-release-branch) above. | |
| # We are utilising single job to wait for all images because this job merely waits | |
| # For the images to be available. | |
| run: breeze prod-image pull --wait-for-image --run-in-parallel | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| if: needs.build-info.outputs.in-workflow-build == 'false' | |
| test-examples-of-prod-image-building: | |
| timeout-minutes: 60 | |
| name: "Test examples of production image building" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-prod-images] | |
| if: needs.build-info.outputs.prod-image-build == 'true' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: > | |
| Pull PROD image | |
| "${{needs.build-info.outputs.default-python-version}}":${{ needs.build-info.outputs.image-tag }} | |
| run: breeze prod-image pull --tag-as-latest | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| - name: "Setup python" | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{needs.build-info.outputs.default-python-version}} | |
| cache: 'pip' | |
| cache-dependency-path: ./dev/requirements.txt | |
| - name: "Test examples of PROD image building" | |
| # yamllint disable-line rule:line-length | |
| run: > | |
| cd ./docker_tests && | |
| python -m pip install -r requirements.txt && | |
| TEST_IMAGE="ghcr.io/${{env.GITHUB_REPOSITORY}}/${{needs.build-info.outputs.default-branch}}/prod/python${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{env.IMAGE_TAG}}" | |
| python -m pytest test_examples_of_prod_image_building.py -n auto --color=yes | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| test-docker-compose-quick-start: | |
| timeout-minutes: 60 | |
| name: "Docker-compose quick start with PROD image verifying" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: [build-info, wait-for-prod-images] | |
| if: needs.build-info.outputs.prod-image-build == 'true' | |
| env: | |
| PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
| INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Install Breeze" | |
| uses: ./.github/actions/breeze | |
| - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| run: breeze prod-image pull --run-in-parallel --tag-as-latest | |
| env: | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| # Force more parallelism for pull even on public images | |
| PARALLELISM: 6 | |
| - name: Verify PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
| run: breeze prod-image verify --run-in-parallel | |
| env: | |
| PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| - name: "Test docker-compose quick start" | |
| run: breeze testing docker-compose-tests | |
| tests-kubernetes: | |
| name: "Kubernetes tests" | |
| uses: ./.github/workflows/k8s-tests.yml | |
| needs: [build-info, wait-for-prod-images] | |
| permissions: | |
| contents: read | |
| packages: read | |
| secrets: inherit | |
| with: | |
| runs-on: ${{ needs.build-info.outputs.runs-on }} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| kubernetes-versions-list-as-string: ${{ needs.build-info.outputs.kubernetes-versions-list-as-string }} | |
| kubernetes-combos-list-as-string: ${{ needs.build-info.outputs.kubernetes-combos-list-as-string }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
| debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
| if: > | |
| ( needs.build-info.outputs.run-kubernetes-tests == 'true' || | |
| needs.build-info.outputs.needs-helm-tests == 'true') | |
| update-constraints: | |
| permissions: | |
| contents: write | |
| packages: write | |
| timeout-minutes: 80 | |
| name: "Update constraints" | |
| runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} | |
| needs: | |
| - build-info | |
| - build-docs | |
| - spellcheck-docs | |
| - wait-for-ci-images | |
| - wait-for-prod-images | |
| - static-checks | |
| - mypy | |
| - tests-sqlite | |
| - tests-mysql | |
| - tests-postgres | |
| - tests-non-db | |
| - tests-special | |
| - tests-integration-postgres | |
| - tests-integration-mysql | |
| - generate-constraints | |
| env: | |
| RUNS_ON: "${{needs.build-info.outputs.runs-on}}" | |
| DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
| PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
| if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' | |
| steps: | |
| - name: "Cleanup repo" | |
| shell: bash | |
| run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
| - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
| uses: actions/checkout@v4 | |
| with: | |
| # Needed to perform push action | |
| persist-credentials: false | |
| - name: Cleanup docker | |
| uses: ./.github/actions/cleanup-docker | |
| - name: "Set constraints branch name" | |
| id: constraints-branch | |
| run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} | |
| - name: Checkout ${{ steps.constraints-branch.outputs.branch }} | |
| uses: actions/checkout@v4 | |
| with: | |
| path: "constraints" | |
| ref: ${{ steps.constraints-branch.outputs.branch }} | |
| persist-credentials: true | |
| fetch-depth: 0 | |
| - name: "Download constraints from the constraints generated by build CI image" | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: constraints | |
| path: ./files | |
| - name: "Diff in constraints for ${{needs.build-info.outputs.python-versions}}" | |
| run: ./scripts/ci/constraints/ci_diff_constraints.sh | |
| # only commit and push constraints in canary runs (main) | |
| - name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}" | |
| run: ./scripts/ci/constraints/ci_commit_constraints.sh | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| - name: "Push changes" | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| working-directory: "constraints" | |
| run: | |
| git push | |
| # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build | |
| # is executed as result of direct push to "main" or one of the "vX-Y-test" branches | |
| # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry | |
| # It will automatically check if a new python image was released and will pull the latest one if needed | |
| push-buildx-cache-to-github-registry: | |
| name: Push Regular Image Cache | |
| needs: [build-info, update-constraints] | |
| uses: ./.github/workflows/push-image-cache.yml | |
| permissions: | |
| contents: read | |
| # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
| # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
| # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the | |
| # "in-workflow-build" condition | |
| packages: write | |
| secrets: inherit | |
| with: | |
| cache-type: "Regular" | |
| include-prod-images: "true" | |
| push-latest-images: "true" | |
| use-uv: "true" | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| if: needs.build-info.outputs.canary-run == 'true' | |
| # This is only a check if ARM images are successfully building when committer runs PR from | |
| # Apache repository. This is needed in case you want to fix failing cache job in "canary" run | |
| # There is no point in running this one in "canary" run, because the above step is doing the | |
| # same build anyway. | |
| build-ci-arm-images: | |
| name: Build CI ARM images (in-workflow) | |
| needs: | |
| - build-info | |
| - build-docs | |
| - spellcheck-docs | |
| - static-checks | |
| - mypy | |
| - tests-sqlite | |
| - tests-mysql | |
| - tests-postgres | |
| - tests-non-db | |
| - tests-integration-postgres | |
| - tests-integration-mysql | |
| uses: ./.github/workflows/ci-image-build.yml | |
| permissions: | |
| contents: read | |
| # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
| # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
| # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the | |
| # "in-workflow-build" condition | |
| packages: write | |
| secrets: inherit | |
| with: | |
| platform: "arm64" | |
| runs-on: ${{needs.build-info.outputs.runs-on}} | |
| image-tag: ${{ needs.build-info.outputs.image-tag }} | |
| python-versions: ${{ needs.build-info.outputs.python-versions }} | |
| branch: ${{ needs.build-info.outputs.default-branch }} | |
| use-uv: "true" | |
| upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
| breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} | |
| constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
| docker-cache: ${{ needs.build-info.outputs.cache-directive }} | |
| if: > | |
| needs.build-info.outputs.in-workflow-build == 'true' && | |
| needs.build-info.outputs.canary-run != 'true' |