Add mypy check to pre-commit-config

.mypy.ini

@@ -0,0 +1,26 @@
+[mypy]
+
+# Specifies the Python version used to parse and check the target program
+python_version = 3.9
+
+# Disallows defining functions without type annotations or with incomplete type annotations
+# True => enforce type annotation in all function definitions
+disallow_untyped_defs = True
+
+# Shows a warning when returning a value with type Any from a function declared with a non- Any return type
+warn_return_any = True
+
+# Shows errors for missing return statements on some execution paths
+warn_no_return = True
+
+# Suppress error messages about imports that cannot be resolved
+# True => ignore all import errors
+ignore_missing_imports = True
+
+# Disallows defining functions with incomplete type annotations
+disallow_incomplete_defs = False
+
+# Directs what to do with imports when the imported module is found as a .py file and not part of the files,
+# modules and packages provided on the command line.
+# SKIP -> mypy checks only single file, not included imports
+follow_imports = skip

.pre-commit-config.yaml

@@ -91,3 +91,9 @@ repos:
         pass_filenames: false
         additional_dependencies:
           - PyYAML == 5.3.1
+      - id: mypy-check
+        name: Check type annotations in python files
+        entry: tools/ci/check_type_comments.py
+        additional_dependencies: ['mypy==0.800', 'mypy-extensions==0.4.3']
+        language: python
+        types: [python]

components/espcoredump/espcoredump.py

@@ -16,6 +16,12 @@
 from corefile.loader import ESPCoreDumpFileLoader, ESPCoreDumpFlashLoader
 from pygdbmi.gdbcontroller import DEFAULT_GDB_TIMEOUT_SEC
 
+try:
+    from typing import Optional, Tuple
+except ImportError:
+    # Only used for type annotations
+    pass
+
 IDF_PATH = os.getenv('IDF_PATH')
 if not IDF_PATH:
     sys.stderr.write('IDF_PATH is not found! Set proper IDF_PATH in environment.\n')
@@ -34,7 +40,7 @@
     CLOSE_FDS = True
 
 
-def load_aux_elf(elf_path):  # type: (str) -> (ElfFile, str)
+def load_aux_elf(elf_path):  # type: (str) -> Tuple[ElfFile, str]
     """
     Loads auxiliary ELF file and composes GDB command to read its symbols.
     """
@@ -48,7 +54,7 @@ def load_aux_elf(elf_path):  # type: (str) -> (ElfFile, str)
     return elf, sym_cmd
 
 
-def core_prepare():
+def core_prepare():  # type: () -> Tuple[Optional[str], ESPCoreDumpFlashLoader]
     loader = None
     core_filename = None
     if not args.core:
@@ -72,7 +78,7 @@ def core_prepare():
     return core_filename, loader
 
 
-def dbg_corefile():
+def dbg_corefile():  # type: () -> None
     """
     Command to load core dump from file or flash and run GDB debug session with it
     """
@@ -91,7 +97,7 @@ def dbg_corefile():
     print('Done!')
 
 
-def info_corefile():
+def info_corefile():  # type: () -> None
     """
     Command to load core dump from file or flash and print it's data in user friendly form
     """

tools/build_apps.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+
 # coding=utf-8
 #
 # ESP-IDF helper script to build multiple applications. Consumes the input of find_apps.py.
@@ -12,7 +13,7 @@
 from find_build_apps.common import SIZE_JSON_FN, rmdir
 
 
-def main():
+def main():  # type: () -> None
     parser = argparse.ArgumentParser(description='ESP-IDF app builder')
     parser.add_argument(
         '-v',

tools/ci/check_type_comments.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+#
+# Copyright 2021 Espressif Systems (Shanghai) CO LTD
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import re
+import subprocess
+from sys import exit
+
+try:
+    from typing import List
+except ImportError:
+    # Only used for type annotations
+    pass
+
+IGNORE_LIST_MYPY = 'tools/ci/mypy_ignore_list.txt'
+COMMENT_RE = re.compile(r'#\s+type:\s+\(', re.MULTILINE)  # REGEX: "# type: ("
+
+
+def type_comments_in_file(file_name):  # type: (str) -> bool
+    """ Check if type annotations (type comments) are present in the file """
+    with open(file_name, 'r') as f:
+        return bool(COMMENT_RE.search(f.read()))
+
+
+def types_valid(file_name):  # type: (str) -> bool
+    """  Run Mypy check on the given file, return TRUE if Mypy check passes """
+    mypy_exit_code = subprocess.call(('mypy %s' % file_name), shell=True)
+    return not bool(mypy_exit_code)
+
+
+def check_files(files):  # type: (List[str]) -> List[str]
+    """
+    Check files for type annotatins:
+    - new python file -> run Mypy check
+    - existed file updated by type annotations -> run Mypy check, remove from ignore list
+    - existed file updated, but no type annotations added -> skip Mypy check on file
+    """
+    type_issues = []
+
+    with open(IGNORE_LIST_MYPY, 'r') as f:
+        ignore_list = [item.strip() for item in f.readlines()]
+        updated_list = ignore_list.copy()
+
+    for file_name in files:
+        if file_name in ignore_list:
+            if type_comments_in_file(file_name):
+                if types_valid(file_name):
+                    updated_list.remove(file_name)
+                    print('\33[93m\n File %s removed automatically from ignore list - run commit again! \n\33[0m' % file_name)
+                else:
+                    type_issues.append(file_name)
+        else:
+            if not types_valid(file_name):
+                type_issues.append(file_name)
+
+    ignore_list.sort()
+    updated_list.sort()
+
+    if updated_list != ignore_list:
+        with open(IGNORE_LIST_MYPY, 'w') as f:
+            for item in updated_list:
+                f.write('%s\n' % item)
+    return type_issues
+
+
+def main():  # type: () -> None
+    parser = argparse.ArgumentParser()
+    parser.add_argument('filenames', nargs='*', help='Filenames to check.')
+    args = parser.parse_args()
+
+    type_issues = check_files(args.filenames)
+
+    if type_issues:
+        print('mypy check failed for:')
+        for file_name in type_issues:
+            print('\t', file_name)
+        exit(1)
+
+
+if __name__ == '__main__':
+    main()

tools/ci/ci_get_mr_info.py

@@ -24,8 +24,14 @@
 
 from gitlab_api import Gitlab
 
+try:
+    from typing import Any, Union
+except ImportError:
+    # Only used for type annotations
+    pass
 
-def _get_mr_obj(source_branch):
+
+def _get_mr_obj(source_branch):  # type: (str) -> Union[Gitlab, None]
     if not source_branch:
         return None
     gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf'))
@@ -46,7 +52,7 @@ def get_mr_iid(source_branch):  # type: (str) -> str
         return str(mr.iid)
 
 
-def get_mr_changed_files(source_branch):
+def get_mr_changed_files(source_branch):  # type: (str) -> Any
     mr = _get_mr_obj(source_branch)
     if not mr:
         return ''
@@ -55,7 +61,7 @@ def get_mr_changed_files(source_branch):
                                     'origin/{}...origin/{}'.format(mr.target_branch, source_branch)]).decode('utf8')
 
 
-def get_mr_commits(source_branch):
+def get_mr_commits(source_branch):  # type: (str) -> str
     mr = _get_mr_obj(source_branch)
     if not mr:
         return ''

tools/ci/executable-list.txt

@@ -48,6 +48,7 @@ tools/ci/check_readme_links.py
 tools/ci/check_rom_apis.sh
 tools/ci/check_rules_yml.py
 tools/ci/check_tools_files_patterns.py
+tools/ci/check_type_comments.py
 tools/ci/check_ut_cmake_make.sh
 tools/ci/checkout_project_ref.py
 tools/ci/deploy_docs.py