Simplify how mypy "folder" checks are run

.github/workflows/ci.yml

@@ -104,7 +104,7 @@ jobs:
       ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
       prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
       docs-build: ${{ steps.selective-checks.outputs.docs-build }}
-      mypy-packages: ${{ steps.selective-checks.outputs.mypy-packages }}
+      mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }}
       needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }}
       needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
       needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
@@ -561,7 +561,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        mypy-package: ${{fromJson(needs.build-info.outputs.mypy-packages)}}
+        mypy-folder: ${{fromJson(needs.build-info.outputs.mypy-folders)}}
     env:
       RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
       PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
@@ -589,17 +589,17 @@ jobs:
           restore-keys: |
             pre-commit-${{steps.breeze.outputs.host-python-version}}-
         if: needs.build-info.outputs.needs-mypy == 'true'
-      - name: "MyPy checks for ${{ matrix.mypy-package }}"
+      - name: "MyPy checks for ${{ matrix.mypy-folder }}"
         run: |
           pip install pre-commit
-          pre-commit run --color always --verbose --hook-stage manual mypy --all-files
+          pre-commit run --color always --verbose --hook-stage manual mypy-${{matrix.mypy-folder}} --all-files
         env:
           VERBOSE: "false"
           COLUMNS: "250"
           SKIP_GROUP_OUTPUT: "true"
           DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
           RUFF_FORMAT: "github"
-          MYPY_PACKAGES: ${{ matrix.mypy-package }}
+
         if: needs.build-info.outputs.needs-mypy == 'true'
 
   # Those checks are run if no image needs to be built for checks. This is for simple changes that

.pre-commit-config.yaml

@@ -1093,14 +1093,32 @@ repos:
         files: ^dev/.*\.py$|^scripts/.*\.py$
         require_serial: true
         additional_dependencies: ['rich>=12.4.4']
-      - id: mypy-core
-        name: Run mypy for core
+      - id: mypy-dev
+        stages: [ 'manual' ]
+        name: Run mypy for dev (manual)
+        language: python
+        entry: ./scripts/ci/pre_commit/pre_commit_mypy_folder.py dev
+        pass_filenames: false
+        files: ^.*\.py$
+        require_serial: true
+        additional_dependencies: [ 'rich>=12.4.4' ]
+      - id: mypy-airflow
+        name: Run mypy for airflow
         language: python
         entry: ./scripts/ci/pre_commit/pre_commit_mypy.py --namespace-packages
         files: \.py$
         exclude: ^.*/.*_vendor/|^airflow/migrations|^airflow/providers|^dev|^scripts|^docs|^provider_packages|^tests/providers|^tests/system/providers|^tests/dags/test_imports.py
         require_serial: true
         additional_dependencies: ['rich>=12.4.4']
+      - id: mypy-airflow
+        stages: [ 'manual' ]
+        name: Run mypy for airflow (manual)
+        language: python
+        entry: ./scripts/ci/pre_commit/pre_commit_mypy_folder.py airflow
+        pass_filenames: false
+        files: ^.*\.py$
+        require_serial: true
+        additional_dependencies: [ 'rich>=12.4.4' ]
       - id: mypy-providers
         name: Run mypy for providers
         language: python
@@ -1109,6 +1127,15 @@ repos:
         exclude: ^.*/.*_vendor/
         require_serial: true
         additional_dependencies: ['rich>=12.4.4']
+      - id: mypy-providers
+        stages: ['manual']
+        name: Run mypy for providers (manual)
+        language: python
+        entry: ./scripts/ci/pre_commit/pre_commit_mypy_folder.py airflow/providers
+        pass_filenames: false
+        files: ^.*\.py$
+        require_serial: true
+        additional_dependencies: ['rich>=12.4.4']
       - id: mypy-docs
         name: Run mypy for /docs/ folder
         language: python
@@ -1117,11 +1144,11 @@ repos:
         exclude: ^docs/rtd-deprecation
         require_serial: true
         additional_dependencies: ['rich>=12.4.4']
-      - id: mypy
+      - id: mypy-docs
         stages: ['manual']
-        name: Run mypy for specified packages (manual)
+        name: Run mypy for /docs/ folder (manual)
         language: python
-        entry: ./scripts/ci/pre_commit/pre_commit_mypy.py
+        entry: ./scripts/ci/pre_commit/pre_commit_mypy_folder.py docs
         pass_filenames: false
         files: ^.*\.py$
         require_serial: true

STATIC_CODE_CHECKS.rst

@@ -136,21 +136,28 @@ require Breeze Docker image to be built locally.
 
 .. note:: Mypy checks
 
-  When we run mypy checks locally when committing a change, one of the ``mypy-*`` checks is run, ``mypy-core``,
+  When we run mypy checks locally when committing a change, one of the ``mypy-*`` checks is run, ``mypy-airflow``,
   ``mypy-dev``, ``mypy-providers``, ``mypy-docs``, depending on the files you are changing. The mypy checks
   are run by passing those changed files to mypy. This is way faster than running checks for all files (even
   if mypy cache is used - especially when you change a file in airflow core that is imported and used by many
   files). However, in some cases, it produces different results than when running checks for the whole set
   of files, because ``mypy`` does not even know that some types are defined in other files and it might not
   be able to follow imports properly if they are dynamic. Therefore in CI we run ``mypy`` check for whole
-  directories (``airflow`` - excluding providers, ``airflow/providers``, ``dev`` and ``docs``) to make sure
+  directories (``airflow`` - excluding providers, ``providers``, ``dev`` and ``docs``) to make sure
   that we catch all ``mypy`` errors - so you can experience different results when running mypy locally and
   in CI. If you want to run mypy checks for all files locally, you can do it by running the following
   command (example for ``airflow`` files):
 
      .. code-block:: bash
 
-        MYPY_PACKAGES="airflow" pre-commit run --hook-stage manual mypy --all-files
+        pre-commit run --hook-stage manual mypy-<FOLDER> --all-files
+
+  For example:
+
+     .. code-block:: bash
+
+        pre-commit run --hook-stage manual mypy-airflow --all-files
+
 
 .. note:: Mypy volume cache
 
@@ -342,15 +349,17 @@ require Breeze Docker image to be built locally.
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | mixed-line-ending                                         | Detect if mixed line ending is used (\r vs. \r\n)            |         |
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+
-| mypy                                                      | Run mypy for specified packages (manual)                     | *       |
-+-----------------------------------------------------------+--------------------------------------------------------------+---------+
-| mypy-core                                                 | Run mypy for core                                            | *       |
+| mypy-airflow                                              | * Run mypy for airflow                                       | *       |
+|                                                           | * Run mypy for airflow (manual)                              |         |
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+
-| mypy-dev                                                  | Run mypy for dev                                             | *       |
+| mypy-dev                                                  | * Run mypy for dev                                           | *       |
+|                                                           | * Run mypy for dev (manual)                                  |         |
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+
-| mypy-docs                                                 | Run mypy for /docs/ folder                                   | *       |
+| mypy-docs                                                 | * Run mypy for /docs/ folder                                 | *       |
+|                                                           | * Run mypy for /docs/ folder (manual)                        |         |
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+
-| mypy-providers                                            | Run mypy for providers                                       | *       |
+| mypy-providers                                            | * Run mypy for providers                                     | *       |
+|                                                           | * Run mypy for providers (manual)                            |         |
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | pretty-format-json                                        | Format JSON files                                            |         |
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+

dev/breeze/src/airflow_breeze/pre_commit_ids.py

@@ -103,8 +103,7 @@
     "lint-markdown",
     "lint-openapi",
     "mixed-line-ending",
-    "mypy",
-    "mypy-core",
+    "mypy-airflow",
     "mypy-dev",
     "mypy-docs",
     "mypy-providers",

dev/breeze/src/airflow_breeze/utils/selective_checks.py

@@ -561,41 +561,41 @@ def _should_be_run(self, source_area: FileGroupForCi) -> bool:
             return False
 
     @cached_property
-    def mypy_packages(self) -> list[str]:
-        packages_to_run: list[str] = []
+    def mypy_folders(self) -> list[str]:
+        folders_to_check: list[str] = []
         if (
             self._matching_files(
                 FileGroupForCi.ALL_AIRFLOW_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES
             )
             or self.full_tests_needed
         ):
-            packages_to_run.append("airflow")
+            folders_to_check.append("airflow")
         if (
             self._matching_files(
                 FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES
             )
             or self._are_all_providers_affected()
         ) and self._default_branch == "main":
-            packages_to_run.append("airflow/providers")
+            folders_to_check.append("providers")
         if (
             self._matching_files(
                 FileGroupForCi.ALL_DOCS_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES
             )
             or self.full_tests_needed
         ):
-            packages_to_run.append("docs")
+            folders_to_check.append("docs")
         if (
             self._matching_files(
                 FileGroupForCi.ALL_DEV_PYTHON_FILES, CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES
             )
             or self.full_tests_needed
         ):
-            packages_to_run.append("dev")
-        return packages_to_run
+            folders_to_check.append("dev")
+        return folders_to_check
 
     @cached_property
     def needs_mypy(self) -> bool:
-        return self.mypy_packages != []
+        return self.mypy_folders != []
 
     @cached_property
     def needs_python_scans(self) -> bool: