From a10130cd0e3b23825587a08f2364b581a11835b2 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Sat, 19 Oct 2024 04:07:09 +0100 Subject: [PATCH] Fix Selective checks for Task SDK Task SDK tests were running as part of "DB tests" with other Core files & providers, this PR changes it so they run separately. It also adds separate mypy checks for it. This commit fixes several other issues to allow running Task SDK tests separately. --- .github/workflows/ci.yml | 22 +- .github/workflows/static-checks-mypy-docs.yml | 8 +- .github/workflows/task-sdk-tests.yml | 14 +- .pre-commit-config.yaml | 18 ++ Dockerfile.ci | 1 + contributing-docs/08_static_code_checks.rst | 3 + dev/breeze/doc/ci/04_selective_checks.md | 2 +- dev/breeze/doc/images/output_shell.svg | 4 +- dev/breeze/doc/images/output_shell.txt | 2 +- .../doc/images/output_static-checks.svg | 4 +- .../doc/images/output_static-checks.txt | 2 +- .../doc/images/output_testing_db-tests.svg | 6 +- .../doc/images/output_testing_db-tests.txt | 2 +- .../images/output_testing_non-db-tests.svg | 6 +- .../images/output_testing_non-db-tests.txt | 2 +- .../doc/images/output_testing_tests.svg | 8 +- .../doc/images/output_testing_tests.txt | 2 +- .../src/airflow_breeze/global_constants.py | 1 - .../src/airflow_breeze/pre_commit_ids.py | 1 + .../airflow_breeze/utils/selective_checks.py | 36 +-- dev/breeze/tests/test_selective_checks.py | 249 +++++++++++------- pyproject.toml | 1 + scripts/ci/pre_commit/mypy_folder.py | 15 +- task_sdk/README.md | 4 + task_sdk/pyproject.toml | 2 +- 25 files changed, 262 insertions(+), 153 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a9d716cd8421..fdf63640af9a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,7 +99,7 @@ jobs: ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} docs-build: ${{ steps.selective-checks.outputs.docs-build }} - mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }} + mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }} needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }} needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} @@ -298,7 +298,7 @@ jobs: runs-on-as-json-docs-build: ${{ needs.build-info.outputs.runs-on-as-json-docs-build }} image-tag: ${{ needs.build-info.outputs.image-tag }} needs-mypy: ${{ needs.build-info.outputs.needs-mypy }} - mypy-folders: ${{ needs.build-info.outputs.mypy-folders }} + mypy-checks: ${{ needs.build-info.outputs.mypy-checks }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} branch: ${{ needs.build-info.outputs.default-branch }} canary-run: ${{ needs.build-info.outputs.canary-run }} @@ -644,6 +644,24 @@ jobs: ( needs.build-info.outputs.run-kubernetes-tests == 'true' || needs.build-info.outputs.needs-helm-tests == 'true') + tests-task-sdk: + name: "Task SDK tests" + uses: ./.github/workflows/task-sdk-tests.yml + needs: [build-info, wait-for-ci-images] + permissions: + contents: read + packages: read + secrets: inherit + with: + runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} + image-tag: ${{ needs.build-info.outputs.image-tag }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + run-task-sdk-tests: ${{ needs.build-info.outputs.run-task-sdk-tests }} + if: > + ( needs.build-info.outputs.run-task-sdk-tests == 'true' || + needs.build-info.outputs.run-tests == 'true') + finalize-tests: name: Finalize tests permissions: diff --git a/.github/workflows/static-checks-mypy-docs.yml b/.github/workflows/static-checks-mypy-docs.yml index c875c1667cf9e..7286c3cf9bca2 100644 --- a/.github/workflows/static-checks-mypy-docs.yml +++ b/.github/workflows/static-checks-mypy-docs.yml @@ -36,7 +36,7 @@ on: # yamllint disable-line rule:truthy description: "Whether to run mypy checks (true/false)" required: true type: string - mypy-folders: + mypy-checks: description: "List of folders to run mypy checks on" required: false type: string @@ -148,7 +148,7 @@ jobs: strategy: fail-fast: false matrix: - mypy-folder: ${{ fromJSON(inputs.mypy-folders) }} + mypy-check: ${{ fromJSON(inputs.mypy-checks) }} env: PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}" IMAGE_TAG: "${{ inputs.image-tag }}" @@ -166,10 +166,10 @@ jobs: - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}" uses: ./.github/actions/prepare_breeze_and_image id: breeze - - name: "MyPy checks for ${{ matrix.mypy-folder }}" + - name: "MyPy checks for ${{ matrix.mypy-check }}" run: | pip install pre-commit - pre-commit run --color always --verbose --hook-stage manual mypy-${{matrix.mypy-folder}} --all-files + pre-commit run --color always --verbose --hook-stage manual ${{matrix.mypy-check}} --all-files env: VERBOSE: "false" COLUMNS: "250" diff --git a/.github/workflows/task-sdk-tests.yml b/.github/workflows/task-sdk-tests.yml index 14fae903837c2..5fa98b935a86b 100644 --- a/.github/workflows/task-sdk-tests.yml +++ b/.github/workflows/task-sdk-tests.yml @@ -28,10 +28,6 @@ on: # yamllint disable-line rule:truthy description: "Tag to set for the image" required: true type: string - canary-run: - description: "Whether this is a canary run" - required: true - type: string default-python-version: description: "Which version of python should be used by default" required: true @@ -40,6 +36,10 @@ on: # yamllint disable-line rule:truthy description: "JSON-formatted array of Python versions to build images from" required: true type: string + run-task-sdk-tests: + description: "Whether to run Task SDK tests or not (true/false)" + required: true + type: string jobs: task-sdk-tests: timeout-minutes: 80 @@ -57,8 +57,8 @@ jobs: INCLUDE_NOT_READY_PROVIDERS: "true" PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" VERBOSE: "true" + MOUNT_SOURCES: "skip" CLEAN_AIRFLOW_INSTALLATION: "${{ inputs.canary-run }}" - if: inputs.run-task-sdk-tests == 'true' steps: - name: "Cleanup repo" shell: bash @@ -81,9 +81,7 @@ jobs: pipx uninstall twine || true pipx install twine && twine check dist/*.whl - name: > - Run provider unit tests on - Airflow Task SDK:Python ${{ matrix.python-version }} - if: matrix.run-tests == 'true' + Run unit tests for Airflow Task SDK:Python ${{ matrix.python-version }} run: > breeze testing tests --run-in-parallel --parallel-test-types TaskSDK diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 29762c19136c2..c3c35e4de66ed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1291,6 +1291,7 @@ repos: ^.*/.*_vendor/ | ^airflow/migrations | ^providers/ | + ^task_sdk/ | ^dev | ^scripts | ^docs | @@ -1343,6 +1344,23 @@ repos: files: ^.*\.py$ require_serial: true additional_dependencies: ['rich>=12.4.4'] + - id: mypy-task-sdk + name: Run mypy for Task SDK + language: python + entry: ./scripts/ci/pre_commit/mypy.py --namespace-packages + files: ^task_sdk/src/airflow/sdk/.*\.py$|^task_sdk/tests//.*\.py$ + exclude: ^.*/.*_vendor/ + require_serial: true + additional_dependencies: ['rich>=12.4.4'] + - id: mypy-task-sdk + stages: ['manual'] + name: Run mypy for Task SDK (manual) + language: python + entry: ./scripts/ci/pre_commit/mypy_folder.py task_sdk/src/airflow/sdk + pass_filenames: false + files: ^.*\.py$ + require_serial: true + additional_dependencies: ['rich>=12.4.4'] - id: check-provider-yaml-valid name: Validate provider.yaml files entry: ./scripts/ci/pre_commit/check_provider_yaml_files.py diff --git a/Dockerfile.ci b/Dockerfile.ci index 826d7109ed551..3ddba289a807f 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1385,6 +1385,7 @@ RUN bash /scripts/docker/install_packaging_tools.sh; \ COPY pyproject.toml ${AIRFLOW_SOURCES}/pyproject.toml COPY providers/pyproject.toml ${AIRFLOW_SOURCES}/providers/pyproject.toml COPY task_sdk/pyproject.toml ${AIRFLOW_SOURCES}/task_sdk/pyproject.toml +COPY task_sdk/README.md ${AIRFLOW_SOURCES}/task_sdk/README.md COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/ COPY tests_common/ ${AIRFLOW_SOURCES}/tests_common/ COPY generated/* ${AIRFLOW_SOURCES}/generated/ diff --git a/contributing-docs/08_static_code_checks.rst b/contributing-docs/08_static_code_checks.rst index f064a13a773b6..aa9955da1afdd 100644 --- a/contributing-docs/08_static_code_checks.rst +++ b/contributing-docs/08_static_code_checks.rst @@ -325,6 +325,9 @@ require Breeze Docker image to be built locally. | mypy-providers | * Run mypy for providers | * | | | * Run mypy for providers (manual) | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ +| mypy-task-sdk | * Run mypy for Task SDK | * | +| | * Run mypy for Task SDK (manual) | | ++-----------------------------------------------------------+--------------------------------------------------------+---------+ | pretty-format-json | Format JSON files | | +-----------------------------------------------------------+--------------------------------------------------------+---------+ | pylint | pylint | | diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index e5894b0296875..5434b1aad4f51 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -203,7 +203,7 @@ Github Actions to pass the list of parameters to a command to execute | kubernetes-combos-list-as-string | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.9-v1.25.2 3.9-v1.26.4 | * | | kubernetes-versions | All Kubernetes versions to use for tests as JSON array | ['v1.25.2'] | | | kubernetes-versions-list-as-string | All Kubernetes versions to use for tests as space-separated string | v1.25.2 | * | -| mypy-folders | List of folders to be considered for mypy | [] | | +| mypy-checks | List of folders to be considered for mypy | [] | | | mysql-exclude | Which versions of MySQL to exclude for tests as JSON array | [] | | | mysql-versions | Which versions of MySQL to use for tests as JSON array | ['5.7'] | | | needs-api-codegen | Whether "api-codegen" are needed to run ("true"/"false") | true | | diff --git a/dev/breeze/doc/images/output_shell.svg b/dev/breeze/doc/images/output_shell.svg index b9c7c480fbd6b..b8eb1713c7f6c 100644 --- a/dev/breeze/doc/images/output_shell.svg +++ b/dev/breeze/doc/images/output_shell.svg @@ -658,8 +658,8 @@ │(All | Default | API | Always | BranchExternalPython |  │ │BranchPythonVenv | CLI | Core | ExternalPython |        │ │Operators | Other | PlainAsserts | Providers |          │ -│PythonVenv | Serialization | TaskSDK | WWW |            │ -│All-Postgres | All-MySQL | All-Quarantined)             │ +│PythonVenv | Serialization | WWW | All-Postgres |       │ +│All-MySQL | All-Quarantined)                            │ │[default: Default]                                      │ │--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It  │ │can also be version (to install from PyPI), `none`,     │ diff --git a/dev/breeze/doc/images/output_shell.txt b/dev/breeze/doc/images/output_shell.txt index d7a9da7d92579..051dd34cd10f7 100644 --- a/dev/breeze/doc/images/output_shell.txt +++ b/dev/breeze/doc/images/output_shell.txt @@ -1 +1 @@ -7b382a009f0280b761743a0746739b80 +fd70e0f17940f32fbc0579e8f77fc6c4 diff --git a/dev/breeze/doc/images/output_static-checks.svg b/dev/breeze/doc/images/output_static-checks.svg index e3c89b304289b..96a324e22c427 100644 --- a/dev/breeze/doc/images/output_static-checks.svg +++ b/dev/breeze/doc/images/output_static-checks.svg @@ -366,8 +366,8 @@ │identity | insert-license | kubeconform | lint-chart-schema | lint-css |         │ │lint-dockerfile | lint-helm-chart | lint-json-schema | lint-markdown |           │ │lint-openapi | mixed-line-ending | mypy-airflow | mypy-dev | mypy-docs |         │ -│mypy-providers | pretty-format-json | pylint | python-no-log-warn |              │ -│replace-bad-characters | rst-backticks | ruff | ruff-format | shellcheck |       │ +│mypy-providers | mypy-task-sdk | pretty-format-json | pylint | python-no-log-warn│ +│| replace-bad-characters | rst-backticks | ruff | ruff-format | shellcheck |     │ │trailing-whitespace | ts-compile-format-lint-ui | ts-compile-format-lint-www |   │ │update-black-version | update-breeze-cmd-output |                                │ │update-breeze-readme-config-hash | update-build-dependencies |                   │ diff --git a/dev/breeze/doc/images/output_static-checks.txt b/dev/breeze/doc/images/output_static-checks.txt index e917996931d60..b0c56ad6640b1 100644 --- a/dev/breeze/doc/images/output_static-checks.txt +++ b/dev/breeze/doc/images/output_static-checks.txt @@ -1 +1 @@ -08a7e37cd651e4d1eb702cb347d9b061 +b4becd0ef113ac04210350ea8f9f98b9 diff --git a/dev/breeze/doc/images/output_testing_db-tests.svg b/dev/breeze/doc/images/output_testing_db-tests.svg index d9d92ed40f050..708665af52de3 100644 --- a/dev/breeze/doc/images/output_testing_db-tests.svg +++ b/dev/breeze/doc/images/output_testing_db-tests.svg @@ -410,15 +410,15 @@ │--parallel-test-typesSpace separated list of test types used for testing in parallel                   │ │(API | Always | BranchExternalPython | BranchPythonVenv | CLI | Core |            │ │ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv |      │ -│Serialization | TaskSDK | WWW)                                                    │ +│Serialization | WWW)                                                              │ │[default: API Always BranchExternalPython BranchPythonVenv CLI Core ExternalPython│ │Operators Other PlainAsserts Providers[-amazon,google] Providers[amazon]          │ -│Providers[google] PythonVenv Serialization TaskSDK WWW]                           │ +│Providers[google] PythonVenv Serialization WWW]                                   │ │--database-isolationRun airflow in database isolation mode.│ │--excluded-parallel-test-typesSpace separated list of test types that will be excluded from parallel tes runs.  │ │(API | Always | BranchExternalPython | BranchPythonVenv | CLI | Core |            │ │ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv |      │ -│Serialization | TaskSDK | WWW)                                                    │ +│Serialization | WWW)                                                              │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Test options ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--test-timeoutTest timeout in seconds. Set the pytest setup, execution and teardown timeouts to this     │ diff --git a/dev/breeze/doc/images/output_testing_db-tests.txt b/dev/breeze/doc/images/output_testing_db-tests.txt index 6245a387214f2..6027a2d1e666a 100644 --- a/dev/breeze/doc/images/output_testing_db-tests.txt +++ b/dev/breeze/doc/images/output_testing_db-tests.txt @@ -1 +1 @@ -690396dbea7c9b6e018704e1ee7f727d +7b406b63cd4a75aba6ac38b8c0b7431c diff --git a/dev/breeze/doc/images/output_testing_non-db-tests.svg b/dev/breeze/doc/images/output_testing_non-db-tests.svg index daf8061b13e40..da43621dd6931 100644 --- a/dev/breeze/doc/images/output_testing_non-db-tests.svg +++ b/dev/breeze/doc/images/output_testing_non-db-tests.svg @@ -376,14 +376,14 @@ │--parallel-test-typesSpace separated list of test types used for testing in parallel                   │ │(API | Always | BranchExternalPython | BranchPythonVenv | CLI | Core |            │ │ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv |      │ -│Serialization | TaskSDK | WWW)                                                    │ +│Serialization | WWW)                                                              │ │[default: API Always BranchExternalPython BranchPythonVenv CLI Core ExternalPython│ │Operators Other PlainAsserts Providers[-amazon,google] Providers[amazon]          │ -│Providers[google] PythonVenv Serialization TaskSDK WWW]                           │ +│Providers[google] PythonVenv Serialization WWW]                                   │ │--excluded-parallel-test-typesSpace separated list of test types that will be excluded from parallel tes runs.  │ │(API | Always | BranchExternalPython | BranchPythonVenv | CLI | Core |            │ │ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv |      │ -│Serialization | TaskSDK | WWW)                                                    │ +│Serialization | WWW)                                                              │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Test options ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--test-timeoutTest timeout in seconds. Set the pytest setup, execution and teardown timeouts to this     │ diff --git a/dev/breeze/doc/images/output_testing_non-db-tests.txt b/dev/breeze/doc/images/output_testing_non-db-tests.txt index 122c3b1acd145..1080b5a377717 100644 --- a/dev/breeze/doc/images/output_testing_non-db-tests.txt +++ b/dev/breeze/doc/images/output_testing_non-db-tests.txt @@ -1 +1 @@ -dbecb30a3e03c7dffd09d46c16687e62 +2cc222da8b9f31b93b527220b76b48a2 diff --git a/dev/breeze/doc/images/output_testing_tests.svg b/dev/breeze/doc/images/output_testing_tests.svg index 8c8c881915a31..c20e2ef16b243 100644 --- a/dev/breeze/doc/images/output_testing_tests.svg +++ b/dev/breeze/doc/images/output_testing_tests.svg @@ -466,19 +466,19 @@ │`Providers[-amazon,google]`                                                       │ │(All | Default | API | Always | BranchExternalPython | BranchPythonVenv | CLI |   │ │Core | ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv │ -│| Serialization | TaskSDK | WWW | All-Postgres | All-MySQL | All-Quarantined)     │ +│| Serialization | WWW | All-Postgres | All-MySQL | All-Quarantined)               │ │[default: Default]                                                                │ │--parallel-test-typesSpace separated list of test types used for testing in parallel                   │ │(API | Always | BranchExternalPython | BranchPythonVenv | CLI | Core |            │ │ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv |      │ -│Serialization | TaskSDK | WWW)                                                    │ +│Serialization | WWW)                                                              │ │[default: API Always BranchExternalPython BranchPythonVenv CLI Core ExternalPython│ │Operators Other PlainAsserts Providers[-amazon,google] Providers[amazon]          │ -│Providers[google] PythonVenv Serialization TaskSDK WWW]                           │ +│Providers[google] PythonVenv Serialization WWW]                                   │ │--excluded-parallel-test-typesSpace separated list of test types that will be excluded from parallel tes runs.  │ │(API | Always | BranchExternalPython | BranchPythonVenv | CLI | Core |            │ │ExternalPython | Operators | Other | PlainAsserts | Providers | PythonVenv |      │ -│Serialization | TaskSDK | WWW)                                                    │ +│Serialization | WWW)                                                              │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Test options ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--test-timeoutTest timeout in seconds. Set the pytest setup, execution and teardown timeouts to this   │ diff --git a/dev/breeze/doc/images/output_testing_tests.txt b/dev/breeze/doc/images/output_testing_tests.txt index ce894444f7f3b..d19f6cf5abdae 100644 --- a/dev/breeze/doc/images/output_testing_tests.txt +++ b/dev/breeze/doc/images/output_testing_tests.txt @@ -1 +1 @@ -c131a2a8ab980041a1a0f5e91fe58ea8 +15002aa129ce25039921f800fb1cf744 diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 00cb24f61051e..35db4d4e530ab 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -194,7 +194,6 @@ class SelectiveUnitTestTypes(Enum): PLAIN_ASSERTS = "PlainAsserts" PROVIDERS = "Providers" PYTHON_VENV = "PythonVenv" - TASK_SDK = "TaskSDK" WWW = "WWW" diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index e684acd24ad2d..91b3ad06330ac 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -117,6 +117,7 @@ "mypy-dev", "mypy-docs", "mypy-providers", + "mypy-task-sdk", "pretty-format-json", "pylint", "python-no-log-warn", diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 3b16b51d59585..90baa5a7c78b2 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -88,12 +88,12 @@ "API Always BranchExternalPython BranchPythonVenv " "CLI Core ExternalPython Operators Other PlainAsserts " "Providers[-amazon,google] Providers[amazon] Providers[google] " - "PythonVenv Serialization TaskSDK WWW" + "PythonVenv Serialization WWW" ) ALL_CI_SELECTIVE_TEST_TYPES_WITHOUT_PROVIDERS = ( "API Always BranchExternalPython BranchPythonVenv CLI Core " - "ExternalPython Operators Other PlainAsserts PythonVenv Serialization TaskSDK WWW" + "ExternalPython Operators Other PlainAsserts PythonVenv Serialization WWW" ) ALL_PROVIDERS_SELECTIVE_TEST_TYPES = "Providers[-amazon,google] Providers[amazon] Providers[google]" @@ -180,6 +180,7 @@ def __hash__(self): r"^airflow/.*\.py$", r"^chart", r"^providers/src/", + r"^task_sdk/src/", r"^tests/system", r"^CHANGELOG\.txt", r"^airflow/config_templates/config\.yml", @@ -307,10 +308,6 @@ def __hash__(self): r"^airflow/serialization/", r"^tests/serialization/", ], - SelectiveUnitTestTypes.TASK_SDK: [ - r"^task_sdk/src/airflow/sdk/", - r"^task_sdk/tests/", - ], SelectiveUnitTestTypes.PYTHON_VENV: PYTHON_OPERATOR_FILES, SelectiveUnitTestTypes.BRANCH_PYTHON_VENV: PYTHON_OPERATOR_FILES, SelectiveUnitTestTypes.EXTERNAL_PYTHON: PYTHON_OPERATOR_FILES, @@ -646,41 +643,46 @@ def _should_be_run(self, source_area: FileGroupForCi) -> bool: return False @cached_property - def mypy_folders(self) -> list[str]: - folders_to_check: list[str] = [] + def mypy_checks(self) -> list[str]: + checks_to_run: list[str] = [] if ( self._matching_files( FileGroupForCi.ALL_AIRFLOW_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES ) or self.full_tests_needed ): - folders_to_check.append("airflow") + checks_to_run.append("mypy-airflow") if ( self._matching_files( FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES ) or self._are_all_providers_affected() ) and self._default_branch == "main": - folders_to_check.append("providers") + checks_to_run.append("mypy-providers") if ( self._matching_files( FileGroupForCi.ALL_DOCS_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES ) or self.full_tests_needed ): - folders_to_check.append("docs") + checks_to_run.append("mypy-docs") if ( self._matching_files( FileGroupForCi.ALL_DEV_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES ) or self.full_tests_needed ): - folders_to_check.append("dev") - return folders_to_check + checks_to_run.append("mypy-dev") + if ( + self._matching_files(FileGroupForCi.TASK_SDK_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES) + or self.full_tests_needed + ): + checks_to_run.append("mypy-task-sdk") + return checks_to_run @cached_property def needs_mypy(self) -> bool: - return self.mypy_folders != [] + return self.mypy_checks != [] @cached_property def needs_python_scans(self) -> bool: @@ -820,6 +822,8 @@ def _get_test_types_to_run(self, split_to_individual_providers: bool = False) -> f"into Core/Other category[/]" ) get_console().print(remaining_files) + if self.run_task_sdk_tests: + candidate_test_types.add("PythonVenv") candidate_test_types.update(all_selective_test_types_except_providers()) else: if "Providers" in candidate_test_types or "API" in candidate_test_types: @@ -1078,7 +1082,9 @@ def skip_pre_commits(self) -> str: # whole package rather than for individual files. That's why we skip those checks in CI # and run them via `mypy-all` command instead and dedicated CI job in matrix # This will also speed up static-checks job usually as the jobs will be running in parallel - pre_commits_to_skip.update({"mypy-providers", "mypy-airflow", "mypy-docs", "mypy-dev"}) + pre_commits_to_skip.update( + {"mypy-providers", "mypy-airflow", "mypy-docs", "mypy-dev", "mypy-task-sdk"} + ) if self._default_branch != "main": # Skip those tests on all "release" branches pre_commits_to_skip.update( diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index ec7f924d2d61f..627779ad0ec82 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -120,13 +120,13 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "false", "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart,mypy-airflow,mypy-dev," - "mypy-docs,mypy-providers,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": None, "providers-test-types-list-as-string": None, "separate-test-types-list-as-string": None, "needs-mypy": "false", - "mypy-folders": "[]", + "mypy-checks": "[]", }, id="No tests on simple change", ) @@ -156,13 +156,13 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev," - "mypy-docs,mypy-providers,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "API Always Providers[common.compat,fab]", "providers-test-types-list-as-string": "Providers[common.compat,fab]", "separate-test-types-list-as-string": "API Always Providers[common.compat] Providers[fab]", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="Only API tests and DOCS and common.compat, FAB providers should run", ) @@ -182,12 +182,12 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev," - "mypy-docs,mypy-providers,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "API Always", "separate-test-types-list-as-string": "API Always", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="Only API tests and DOCS should run (no provider tests) when only internal api changed", ) @@ -207,12 +207,12 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev," - "mypy-docs,mypy-providers,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "API Always", "separate-test-types-list-as-string": "API Always", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="Only API tests and DOCS should run (no provider tests) when only ui api changed", ) @@ -232,12 +232,12 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "false", "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev," - "mypy-docs,mypy-providers,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "API Always", "separate-test-types-list-as-string": "API Always", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="Only API tests should run (no provider tests) and no DOCs build when only test API files changed", ) @@ -258,13 +258,13 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev," - "mypy-docs,mypy-providers,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Always Operators", "providers-test-types-list-as-string": "", "separate-test-types-list-as-string": "Always Operators", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="Only Operator tests and DOCS should run", ) @@ -285,7 +285,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev," - "mypy-docs,mypy-providers,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Always BranchExternalPython BranchPythonVenv " "ExternalPython Operators PythonVenv", @@ -293,7 +293,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "separate-test-types-list-as-string": "Always BranchExternalPython BranchPythonVenv " "ExternalPython Operators PythonVenv", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="Only Python tests", ) @@ -314,13 +314,13 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev," - "mypy-docs,mypy-providers,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Always Serialization", "providers-test-types-list-as-string": "", "separate-test-types-list-as-string": "Always Serialization", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="Only Serialization tests", ) @@ -344,7 +344,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-tests": "true", "run-amazon-tests": "true", "docs-build": "true", - "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "API Always Providers[amazon] " @@ -355,7 +355,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "Providers[fab] Providers[google] Providers[openlineage] Providers[pgvector] " "Providers[postgres]", "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers']", + "mypy-checks": "['mypy-airflow', 'mypy-providers']", }, id="API and providers tests and docs should run", ) @@ -375,15 +375,17 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-tests": "true", "run-amazon-tests": "false", "docs-build": "false", - "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," - "ts-compile-format-lint-ui,ts-compile-format-lint-www", + "skip-pre-commits": ( + "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs" + ",mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www" + ), "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Always Providers[apache.beam] Providers[google]", "providers-test-types-list-as-string": "Providers[apache.beam] Providers[google]", "separate-test-types-list-as-string": "Always Providers[apache.beam] Providers[google]", "needs-mypy": "true", - "mypy-folders": "['providers']", + "mypy-checks": "['mypy-providers']", }, id="Selected Providers and docs should run", ) @@ -404,17 +406,52 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart,mypy-airflow,mypy-dev," - "mypy-docs,mypy-providers,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": None, "providers-test-types-list-as-string": None, "needs-mypy": "false", - "mypy-folders": "[]", + "mypy-checks": "[]", }, id="Only docs builds should run - no tests needed", ) ), + ( + pytest.param( + ("task_sdk/src/airflow/sdk/random.py",), + { + "all-python-versions": "['3.9']", + "all-python-versions-list-as-string": "3.9", + "python-versions": "['3.9']", + "python-versions-list-as-string": "3.9", + "ci-image-build": "true", + "prod-image-build": "false", + "needs-api-tests": "false", + "needs-helm-tests": "false", + "run-kubernetes-tests": "false", + "run-tests": "true", + "run-task-sdk-tests": "true", + "docs-build": "true", + "full-tests-needed": "false", + "skip-pre-commits": ( + "check-provider-yaml-valid,identity,lint-helm-chart" + ",mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk" + ",ts-compile-format-lint-ui,ts-compile-format-lint-www" + ), + "skip-provider-tests": "true", + "upgrade-to-newer-dependencies": "false", + "parallel-test-types-list-as-string": ( + "API Always BranchExternalPython BranchPythonVenv CLI Core ExternalPython " + "Operators Other PlainAsserts PythonVenv Serialization WWW" + ), + "providers-test-types-list-as-string": "", + "needs-mypy": "true", + "mypy-checks": "['mypy-task-sdk']", + }, + id="Task SDK source file changed - Task SDK & Core tests should run", + ) + ), ( pytest.param( ( @@ -434,7 +471,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-tests": "true", "run-amazon-tests": "true", "docs-build": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", @@ -443,7 +480,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "providers-test-types-list-as-string": "Providers[amazon] " "Providers[common.sql,openlineage,pgvector,postgres] Providers[google]", "needs-mypy": "true", - "mypy-folders": "['providers']", + "mypy-checks": "['mypy-providers']", }, id="Helm tests, providers (both upstream and downstream)," "kubernetes tests and docs should run", @@ -469,7 +506,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-tests": "true", "run-amazon-tests": "true", "docs-build": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", @@ -480,7 +517,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "Providers[apache.livy] Providers[dbt.cloud] " "Providers[dingding] Providers[discord] Providers[http]", "needs-mypy": "true", - "mypy-folders": "['providers']", + "mypy-checks": "['mypy-providers']", }, id="Helm tests, http and all relevant providers, kubernetes tests and " "docs should run even if unimportant files were added", @@ -505,14 +542,14 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-tests": "true", "run-amazon-tests": "false", "docs-build": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Always Providers[airbyte]", "providers-test-types-list-as-string": "Providers[airbyte]", "needs-mypy": "true", - "mypy-folders": "['providers']", + "mypy-checks": "['mypy-providers']", }, id="Helm tests, airbyte providers, kubernetes tests and " "docs should run even if unimportant files were added", @@ -537,14 +574,14 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-tests": "true", "docs-build": "true", "skip-pre-commits": "check-provider-yaml-valid,identity,mypy-airflow,mypy-dev," - "mypy-docs,mypy-providers,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-amazon-tests": "false", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Always", "providers-test-types-list-as-string": "", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="Docs should run even if unimportant files were added and prod image " "should be build for chart changes", @@ -566,12 +603,12 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "true", "docs-build": "true", "full-tests-needed": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "upgrade-to-newer-dependencies": "true", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "providers-test-types-list-as-string": ALL_PROVIDERS_SELECTIVE_TEST_TYPES, "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="Everything should run - including all providers and upgrading to " "newer requirements as pyproject.toml changed and all Python versions", @@ -593,12 +630,12 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "true", "docs-build": "true", "full-tests-needed": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "upgrade-to-newer-dependencies": "true", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "providers-test-types-list-as-string": ALL_PROVIDERS_SELECTIVE_TEST_TYPES, "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="Everything should run and upgrading to newer requirements as dependencies change", ) @@ -618,7 +655,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "needs-helm-tests": "false", "run-tests": "true", "docs-build": "true", - "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", @@ -627,7 +664,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "Providers[apache.hive,cncf.kubernetes,common.compat,common.sql,exasol,ftp,http," "imap,microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] Providers[google]", "needs-mypy": "true", - "mypy-folders": "['providers']", + "mypy-checks": "['mypy-providers']", }, id="Providers tests run including amazon tests if amazon provider files changed", ), @@ -645,13 +682,13 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-tests": "true", "run-amazon-tests": "false", "docs-build": "false", - "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Always Providers[airbyte]", "needs-mypy": "true", - "mypy-folders": "['providers']", + "mypy-checks": "['mypy-providers']", }, id="Providers tests run without amazon tests if no amazon file changed", ), @@ -671,7 +708,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-tests": "true", "run-amazon-tests": "true", "docs-build": "true", - "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", @@ -679,7 +716,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "Providers[apache.hive,cncf.kubernetes,common.compat,common.sql,exasol,ftp,http," "imap,microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh,teradata] Providers[google]", "needs-mypy": "true", - "mypy-folders": "['providers']", + "mypy-checks": "['mypy-providers']", }, id="Providers tests run including amazon tests if amazon provider files changed", ), @@ -702,12 +739,12 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "false", "run-kubernetes-tests": "false", - "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Always Providers[common.compat,common.io,openlineage]", "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers']", + "mypy-checks": "['mypy-airflow', 'mypy-providers']", }, id="Only Always and common providers tests should run when only common.io and tests/always changed", ), @@ -726,12 +763,12 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "true", "run-kubernetes-tests": "false", - "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Always Core Providers[standard] Serialization", "needs-mypy": "true", - "mypy-folders": "['providers']", + "mypy-checks": "['mypy-providers']", }, id="Providers standard tests and Serialization tests to run when airflow bash.py changed", ), @@ -750,12 +787,12 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "false", "docs-build": "false", "run-kubernetes-tests": "false", - "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Always Core Providers[standard] Serialization", "needs-mypy": "true", - "mypy-folders": "['providers']", + "mypy-checks": "['mypy-providers']", }, id="Force Core and Serialization tests to run when tests bash changed", ), @@ -775,12 +812,12 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "true", "docs-build": "true", "full-tests-needed": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "providers-test-types-list-as-string": ALL_PROVIDERS_SELECTIVE_TEST_TYPES, "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="All tests should be run when tests/utils/ change", ) @@ -801,12 +838,12 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-amazon-tests": "true", "docs-build": "true", "full-tests-needed": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "providers-test-types-list-as-string": ALL_PROVIDERS_SELECTIVE_TEST_TYPES, "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="All tests should be run when tests_common/ change", ) @@ -972,12 +1009,12 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "docs-build": "true", "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "full-tests-needed": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "providers-test-types-list-as-string": ALL_PROVIDERS_SELECTIVE_TEST_TYPES, "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="Everything should run including all providers when full tests are needed, " "and all versions are required.", @@ -1006,12 +1043,12 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "docs-build": "true", "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "full-tests-needed": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "providers-test-types-list-as-string": ALL_PROVIDERS_SELECTIVE_TEST_TYPES, "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="Everything should run including all providers when full tests are needed " "but with single python and kubernetes if `default versions only` label is set", @@ -1040,12 +1077,12 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "docs-build": "true", "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "full-tests-needed": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "providers-test-types-list-as-string": ALL_PROVIDERS_SELECTIVE_TEST_TYPES, "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="Everything should run including all providers when full tests are needed " "but with single python and kubernetes if no version label is set", @@ -1075,12 +1112,12 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "docs-build": "true", "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "full-tests-needed": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "providers-test-types-list-as-string": ALL_PROVIDERS_SELECTIVE_TEST_TYPES, "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="Everything should run including all providers when full tests are needed " "but with single python and kubernetes if `latest versions only` label is set", @@ -1110,12 +1147,12 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "docs-build": "true", "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "full-tests-needed": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "providers-test-types-list-as-string": ALL_PROVIDERS_SELECTIVE_TEST_TYPES, "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="Everything should run including full providers when full " "tests are needed even with different label set as well", @@ -1142,16 +1179,16 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "docs-build": "true", "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "full-tests-needed": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "providers-test-types-list-as-string": ALL_PROVIDERS_SELECTIVE_TEST_TYPES, "separate-test-types-list-as-string": "API Always BranchExternalPython BranchPythonVenv " "CLI Core ExternalPython Operators Other PlainAsserts " + LIST_OF_ALL_PROVIDER_TESTS - + " PythonVenv Serialization TaskSDK WWW", + + " PythonVenv Serialization WWW", "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="Everything should run including full providers when " "full tests are needed even if no files are changed", @@ -1175,17 +1212,17 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ "docs-build": "true", "docs-list-as-string": "apache-airflow docker-stack", "full-tests-needed": "true", - "skip-pre-commits": "check-airflow-provider-compatibility,check-extra-packages-references,check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,validate-operators-init", + "skip-pre-commits": "check-airflow-provider-compatibility,check-extra-packages-references,check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,validate-operators-init", "skip-provider-tests": "true", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "API Always BranchExternalPython " "BranchPythonVenv CLI Core ExternalPython Operators Other PlainAsserts " - "PythonVenv Serialization TaskSDK WWW", + "PythonVenv Serialization WWW", "separate-test-types-list-as-string": "API Always BranchExternalPython " "BranchPythonVenv CLI Core ExternalPython Operators Other PlainAsserts " - "PythonVenv Serialization TaskSDK WWW", + "PythonVenv Serialization WWW", "needs-mypy": "true", - "mypy-folders": "['airflow', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="Everything should run except Providers and lint pre-commit " "when full tests are needed for non-main branch", @@ -1228,7 +1265,7 @@ def test_expected_output_full_tests_needed( "skip-provider-tests": "true", "parallel-test-types-list-as-string": None, "needs-mypy": "false", - "mypy-folders": "[]", + "mypy-checks": "[]", }, id="Nothing should run if only non-important files changed", ), @@ -1255,7 +1292,7 @@ def test_expected_output_full_tests_needed( "skip-provider-tests": "true", "parallel-test-types-list-as-string": "Always", "needs-mypy": "false", - "mypy-folders": "[]", + "mypy-checks": "[]", }, id="No Helm tests, No providers no lint charts, should run if " "only chart/providers changed in non-main but PROD image should be built", @@ -1285,7 +1322,7 @@ def test_expected_output_full_tests_needed( "skip-provider-tests": "true", "parallel-test-types-list-as-string": "Always CLI", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="Only CLI tests and Kubernetes tests should run if cli/chart files changed in non-main branch", ), @@ -1309,9 +1346,9 @@ def test_expected_output_full_tests_needed( "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "true", "parallel-test-types-list-as-string": "API Always BranchExternalPython BranchPythonVenv " - "CLI Core ExternalPython Operators Other PlainAsserts PythonVenv Serialization TaskSDK WWW", + "CLI Core ExternalPython Operators Other PlainAsserts PythonVenv Serialization WWW", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="All tests except Providers and helm lint pre-commit " "should run if core file changed in non-main branch", @@ -1348,11 +1385,11 @@ def test_expected_output_pull_request_v2_7( "docs-list-as-string": None, "upgrade-to-newer-dependencies": "false", "skip-pre-commits": "check-provider-yaml-valid,flynt,identity,lint-helm-chart," - "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", "skip-provider-tests": "true", "parallel-test-types-list-as-string": None, "needs-mypy": "false", - "mypy-folders": "[]", + "mypy-checks": "[]", }, id="Nothing should run if only non-important files changed", ), @@ -1368,13 +1405,13 @@ def test_expected_output_pull_request_v2_7( "run-tests": "true", "docs-build": "true", "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "true", "parallel-test-types-list-as-string": "Always", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="Only Always and docs build should run if only system tests changed", ), @@ -1400,7 +1437,7 @@ def test_expected_output_pull_request_v2_7( "cncf.kubernetes common.compat common.sql facebook google hashicorp microsoft.azure " "microsoft.mssql mysql openlineage oracle postgres " "presto salesforce samba sftp ssh trino", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-ui,ts-compile-format-lint-www", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "false", @@ -1409,7 +1446,7 @@ def test_expected_output_pull_request_v2_7( "hashicorp,microsoft.azure,microsoft.mssql,mysql,openlineage,oracle,postgres,presto," "salesforce,samba,sftp,ssh,trino] Providers[google]", "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers']", + "mypy-checks": "['mypy-airflow', 'mypy-providers']", }, id="CLI tests and Google-related provider tests should run if cli/chart files changed but " "prod image should be build too and k8s tests too", @@ -1431,14 +1468,14 @@ def test_expected_output_pull_request_v2_7( "run-tests": "true", "docs-build": "true", "docs-list-as-string": "apache-airflow common.compat fab", - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "false", "parallel-test-types-list-as-string": "API Always CLI Operators Providers[common.compat,fab] WWW", "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="No providers tests except common.compat fab should run if only CLI/API/Operators/WWW file changed", ), @@ -1453,14 +1490,14 @@ def test_expected_output_pull_request_v2_7( "run-tests": "true", "docs-build": "true", "docs-list-as-string": "apache-airflow", - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "true", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES_WITHOUT_PROVIDERS, "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="Tests for all airflow core types except providers should run if model file changed", ), @@ -1475,14 +1512,14 @@ def test_expected_output_pull_request_v2_7( "run-tests": "true", "docs-build": "true", "docs-list-as-string": "apache-airflow", - "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers," + "skip-pre-commits": "check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk," "ts-compile-format-lint-ui,ts-compile-format-lint-www", "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "skip-provider-tests": "true", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES_WITHOUT_PROVIDERS, "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, id="Tests for all airflow core types except providers should run if " "any other than API/WWW/CLI/Operators file changed.", @@ -1520,11 +1557,11 @@ def test_expected_output_pull_request_target( "run-tests": "true", "docs-build": "true", "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "upgrade-to-newer-dependencies": "true", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="All tests run on push even if unimportant file changed", ), @@ -1541,13 +1578,13 @@ def test_expected_output_pull_request_target( "needs-helm-tests": "false", "run-tests": "true", "docs-build": "true", - "skip-pre-commits": "check-airflow-provider-compatibility,check-extra-packages-references,check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,validate-operators-init", + "skip-pre-commits": "check-airflow-provider-compatibility,check-extra-packages-references,check-provider-yaml-valid,identity,lint-helm-chart,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk,validate-operators-init", "docs-list-as-string": "apache-airflow docker-stack", "upgrade-to-newer-dependencies": "true", "parallel-test-types-list-as-string": "API Always BranchExternalPython BranchPythonVenv " - "CLI Core ExternalPython Operators Other PlainAsserts PythonVenv Serialization TaskSDK WWW", + "CLI Core ExternalPython Operators Other PlainAsserts PythonVenv Serialization WWW", "needs-mypy": "true", - "mypy-folders": "['airflow', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="All tests except Providers and Helm run on push" " even if unimportant file changed in non-main branch", @@ -1565,12 +1602,12 @@ def test_expected_output_pull_request_target( "needs-helm-tests": "true", "run-tests": "true", "docs-build": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD, "upgrade-to-newer-dependencies": "true", "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, id="All tests run on push if core file changed", ), @@ -1619,13 +1656,13 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): "needs-helm-tests": "true", "run-tests": "true", "docs-build": "true", - "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers", + "skip-pre-commits": "identity,mypy-airflow,mypy-dev,mypy-docs,mypy-providers,mypy-task-sdk", "upgrade-to-newer-dependencies": ( "true" if github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE] else "false" ), "parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, str(stderr), ) @@ -2203,7 +2240,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs ("README.md",), { "needs-mypy": "false", - "mypy-folders": "[]", + "mypy-checks": "[]", }, "main", (), @@ -2213,7 +2250,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs ("airflow/cli/file.py",), { "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, "main", (), @@ -2223,7 +2260,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs ("airflow/models/file.py",), { "needs-mypy": "true", - "mypy-folders": "['airflow']", + "mypy-checks": "['mypy-airflow']", }, "main", (), @@ -2233,17 +2270,27 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs ("providers/src/airflow/providers/a_file.py",), { "needs-mypy": "true", - "mypy-folders": "['providers']", + "mypy-checks": "['mypy-providers']", }, "main", (), id="Airflow mypy checks on provider files", ), + pytest.param( + ("task_sdk/src/airflow/sdk/a_file.py",), + { + "needs-mypy": "true", + "mypy-checks": "['mypy-task-sdk']", + }, + "main", + (), + id="Airflow mypy checks on Task SDK files", + ), pytest.param( ("docs/a_file.py",), { "needs-mypy": "true", - "mypy-folders": "['docs']", + "mypy-checks": "['mypy-docs']", }, "main", (), @@ -2253,7 +2300,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs ("dev/a_package/a_file.py",), { "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, "main", (), @@ -2263,7 +2310,7 @@ def test_provider_compatibility_checks(labels: tuple[str, ...], expected_outputs ("readme.md",), { "needs-mypy": "true", - "mypy-folders": "['airflow', 'providers', 'docs', 'dev']", + "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev', 'mypy-task-sdk']", }, "main", ("full tests needed",), diff --git a/pyproject.toml b/pyproject.toml index f4512a382621f..f23c0afb6749c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -592,6 +592,7 @@ explicit_package_bases = true mypy_path = [ "$MYPY_CONFIG_FILE_DIR", "$MYPY_CONFIG_FILE_DIR/providers/src", + "$MYPY_CONFIG_FILE_DIR/task_sdk/src", ] [[tool.mypy.overrides]] diff --git a/scripts/ci/pre_commit/mypy_folder.py b/scripts/ci/pre_commit/mypy_folder.py index b2d7c76580276..36a7840fff9b8 100755 --- a/scripts/ci/pre_commit/mypy_folder.py +++ b/scripts/ci/pre_commit/mypy_folder.py @@ -31,7 +31,13 @@ initialize_breeze_precommit(__name__, __file__) -ALLOWED_FOLDERS = ["airflow", "providers/src/airflow/providers", "dev", "docs"] +ALLOWED_FOLDERS = [ + "airflow", + "providers/src/airflow/providers", + "dev", + "docs", + "task_sdk/src/airflow/sdk", +] if len(sys.argv) < 2: console.print(f"[yellow]You need to specify the folder to test as parameter: {ALLOWED_FOLDERS}\n") @@ -50,6 +56,13 @@ "--namespace-packages", ] ) +if mypy_folder == "task_sdk/src/airflow/sdk": + arguments.extend( + [ + "task_sdk/tests", + "--namespace-packages", + ] + ) if mypy_folder == "airflow": arguments.extend( diff --git a/task_sdk/README.md b/task_sdk/README.md index ef14affc68c62..d74c50d4d740b 100644 --- a/task_sdk/README.md +++ b/task_sdk/README.md @@ -16,3 +16,7 @@ specific language governing permissions and limitations under the License. --> + +# Apache Airflow Task SDK + +The Apache Airflow Task SDK includes interfaces for DAG authors. diff --git a/task_sdk/pyproject.toml b/task_sdk/pyproject.toml index 149a9731ce994..be2be98baf86e 100644 --- a/task_sdk/pyproject.toml +++ b/task_sdk/pyproject.toml @@ -19,7 +19,7 @@ name = "apache-airflow-task-sdk" version = "0.1.0.dev0" description = "Python Task SDK for Apache Airflow DAG Authors" -#readme = "README.md" +readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.9, <3.13" dependencies = []