Fix Selective checks for Task SDK

Task SDK tests were running as part of "DB tests" with other Core files & providers, this PR changes it so they run separately. It also adds separate mypy checks for it. This commit fixes several other issues to allow running Task SDK tests separately.
apache · Oct 19, 2024 · a10130c · a10130c
1 parent 899dcbf
commit a10130c
Show file tree

Hide file tree

Showing 25 changed files with 262 additions and 153 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -99,7 +99,7 @@ jobs:
       ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
       prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
       docs-build: ${{ steps.selective-checks.outputs.docs-build }}
-      mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }}
+      mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }}
       needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }}
       needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
       needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
@@ -298,7 +298,7 @@ jobs:
       runs-on-as-json-docs-build: ${{ needs.build-info.outputs.runs-on-as-json-docs-build }}
       image-tag: ${{ needs.build-info.outputs.image-tag }}
       needs-mypy: ${{ needs.build-info.outputs.needs-mypy }}
-      mypy-folders: ${{ needs.build-info.outputs.mypy-folders }}
+      mypy-checks: ${{ needs.build-info.outputs.mypy-checks }}
       python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
       branch: ${{ needs.build-info.outputs.default-branch }}
       canary-run: ${{ needs.build-info.outputs.canary-run }}
@@ -644,6 +644,24 @@ jobs:
       ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
       needs.build-info.outputs.needs-helm-tests == 'true')
 
+  tests-task-sdk:
+    name: "Task SDK tests"
+    uses: ./.github/workflows/task-sdk-tests.yml
+    needs: [build-info, wait-for-ci-images]
+    permissions:
+      contents: read
+      packages: read
+    secrets: inherit
+    with:
+      runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
+      image-tag: ${{ needs.build-info.outputs.image-tag }}
+      default-python-version: ${{ needs.build-info.outputs.default-python-version }}
+      python-versions: ${{ needs.build-info.outputs.python-versions }}
+      run-task-sdk-tests: ${{ needs.build-info.outputs.run-task-sdk-tests }}
+    if: >
+      ( needs.build-info.outputs.run-task-sdk-tests == 'true' ||
+      needs.build-info.outputs.run-tests == 'true')
+
   finalize-tests:
     name: Finalize tests
     permissions:

diff --git a/.github/workflows/static-checks-mypy-docs.yml b/.github/workflows/static-checks-mypy-docs.yml
@@ -36,7 +36,7 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to run mypy checks (true/false)"
         required: true
         type: string
-      mypy-folders:
+      mypy-checks:
         description: "List of folders to run mypy checks on"
         required: false
         type: string
@@ -148,7 +148,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        mypy-folder: ${{ fromJSON(inputs.mypy-folders) }}
+        mypy-check: ${{ fromJSON(inputs.mypy-checks) }}
     env:
       PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
       IMAGE_TAG: "${{ inputs.image-tag }}"
@@ -166,10 +166,10 @@ jobs:
       - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}"
         uses: ./.github/actions/prepare_breeze_and_image
         id: breeze
-      - name: "MyPy checks for ${{ matrix.mypy-folder }}"
+      - name: "MyPy checks for ${{ matrix.mypy-check }}"
         run: |
           pip install pre-commit
-          pre-commit run --color always --verbose --hook-stage manual mypy-${{matrix.mypy-folder}} --all-files
+          pre-commit run --color always --verbose --hook-stage manual ${{matrix.mypy-check}} --all-files
         env:
           VERBOSE: "false"
           COLUMNS: "250"

diff --git a/.github/workflows/task-sdk-tests.yml b/.github/workflows/task-sdk-tests.yml
@@ -28,10 +28,6 @@ on:  # yamllint disable-line rule:truthy
         description: "Tag to set for the image"
         required: true
         type: string
-      canary-run:
-        description: "Whether this is a canary run"
-        required: true
-        type: string
       default-python-version:
         description: "Which version of python should be used by default"
         required: true
@@ -40,6 +36,10 @@ on:  # yamllint disable-line rule:truthy
         description: "JSON-formatted array of Python versions to build images from"
         required: true
         type: string
+      run-task-sdk-tests:
+        description: "Whether to run Task SDK tests or not (true/false)"
+        required: true
+        type: string
 jobs:
   task-sdk-tests:
     timeout-minutes: 80
@@ -57,8 +57,8 @@ jobs:
       INCLUDE_NOT_READY_PROVIDERS: "true"
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
       VERBOSE: "true"
+      MOUNT_SOURCES: "skip"
       CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}"
-    if: inputs.run-task-sdk-tests == 'true'
     steps:
       - name: "Cleanup repo"
         shell: bash
@@ -81,9 +81,7 @@ jobs:
           pipx uninstall twine || true
           pipx install twine && twine check dist/*.whl
       - name: >
-          Run provider unit tests on
-          Airflow Task SDK:Python ${{ matrix.python-version }}
-        if: matrix.run-tests == 'true'
+          Run unit tests for Airflow Task SDK:Python ${{ matrix.python-version }}
         run: >
           breeze testing tests --run-in-parallel
           --parallel-test-types TaskSDK

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1291,6 +1291,7 @@ repos:
           ^.*/.*_vendor/ |
           ^airflow/migrations |
           ^providers/ |
+          ^task_sdk/ |
           ^dev |
           ^scripts |
           ^docs |
@@ -1343,6 +1344,23 @@ repos:
         files: ^.*\.py$
         require_serial: true
         additional_dependencies: ['rich>=12.4.4']
+      - id: mypy-task-sdk
+        name: Run mypy for Task SDK
+        language: python
+        entry: ./scripts/ci/pre_commit/mypy.py --namespace-packages
+        files: ^task_sdk/src/airflow/sdk/.*\.py$|^task_sdk/tests//.*\.py$
+        exclude: ^.*/.*_vendor/
+        require_serial: true
+        additional_dependencies: ['rich>=12.4.4']
+      - id: mypy-task-sdk
+        stages: ['manual']
+        name: Run mypy for Task SDK (manual)
+        language: python
+        entry: ./scripts/ci/pre_commit/mypy_folder.py task_sdk/src/airflow/sdk
+        pass_filenames: false
+        files: ^.*\.py$
+        require_serial: true
+        additional_dependencies: ['rich>=12.4.4']
       - id: check-provider-yaml-valid
         name: Validate provider.yaml files
         entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -1385,6 +1385,7 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \
 COPY pyproject.toml ${AIRFLOW_SOURCES}/pyproject.toml
 COPY providers/pyproject.toml ${AIRFLOW_SOURCES}/providers/pyproject.toml
 COPY task_sdk/pyproject.toml ${AIRFLOW_SOURCES}/task_sdk/pyproject.toml
+COPY task_sdk/README.md ${AIRFLOW_SOURCES}/task_sdk/README.md
 COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/
 COPY tests_common/ ${AIRFLOW_SOURCES}/tests_common/
 COPY generated/* ${AIRFLOW_SOURCES}/generated/

diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst
@@ -325,6 +325,9 @@ require Breeze Docker image to be built locally.
 | mypy-providers                                            | * Run mypy for providers                               | *       |
 |                                                           | * Run mypy for providers (manual)                      |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
+| mypy-task-sdk                                             | * Run mypy for Task SDK                                | *       |
+|                                                           | * Run mypy for Task SDK (manual)                       |         |
++-----------------------------------------------------------+--------------------------------------------------------+---------+
 | pretty-format-json                                        | Format JSON files                                      |         |
 +-----------------------------------------------------------+--------------------------------------------------------+---------+
 | pylint                                                    | pylint                                                 |         |

diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md
@@ -203,7 +203,7 @@ Github Actions to pass the list of parameters to a command to execute
 | kubernetes-combos-list-as-string       | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.9-v1.25.2 3.9-v1.26.4                   | *              |
 | kubernetes-versions                    | All Kubernetes versions to use for tests as JSON array                                               | ['v1.25.2']                               |                |
 | kubernetes-versions-list-as-string     | All Kubernetes versions to use for tests as space-separated string                                   | v1.25.2                                   | *              |
-| mypy-folders                           | List of folders to be considered for mypy                                                            | []                                        |                |
+| mypy-checks                           | List of folders to be considered for mypy                                                            | []                                        |                |
 | mysql-exclude                          | Which versions of MySQL to exclude for tests as JSON array                                           | []                                        |                |
 | mysql-versions                         | Which versions of MySQL to use for tests as JSON array                                               | ['5.7']                                   |                |
 | needs-api-codegen                      | Whether "api-codegen" are needed to run ("true"/"false")                                             | true                                      |                |

diff --git a/dev/breeze/doc/images/output_shell.svg b/dev/breeze/doc/images/output_shell.svg
diff --git a/dev/breeze/doc/images/output_shell.txt b/dev/breeze/doc/images/output_shell.txt
@@ -1 +1 @@
-7b382a009f0280b761743a0746739b80
+fd70e0f17940f32fbc0579e8f77fc6c4