🔧 Make type-checking opt-out (#5811)

This commit makes mypy type-checking of files opt-in,
by replacing `files` with `exclude` in the pre-commit config.
It also increases the strictness of the type-checking,
primarily by removing the `follow_imports = "skip"` config
for aiida submodules imports in the `pyproject.toml`.
Numerous fixes have also been made,
to reduce the number of excluded files,
only where fixes did not affect any code behaviour.

.pre-commit-config.yaml

@@ -70,54 +70,138 @@ repos:
         types: [python]
         require_serial: true
         pass_filenames: true
-        files: >-
+        exclude: >-
             (?x)^(
-                aiida/cmdline/commands/cmd_rabbitmq.py|
-                aiida/common/timezone.py|
-                aiida/common/progress_reporter.py|
-                aiida/engine/.*py|
-                aiida/manage/manager.py|
-                aiida/manage/database/delete/nodes.py|
-                aiida/orm/querybuilder.py|
-                aiida/orm/implementation/entities.py|
-                aiida/orm/implementation/storage_backend.py|
-                aiida/orm/implementation/authinfos.py|
-                aiida/orm/implementation/comments.py|
-                aiida/orm/implementation/computers.py|
-                aiida/orm/implementation/groups.py|
-                aiida/orm/implementation/logs.py|
-                aiida/orm/implementation/nodes.py|
-                aiida/orm/implementation/users.py|
-                aiida/orm/implementation/querybuilder.py|
-                aiida/orm/entities.py|
-                aiida/orm/extras.py|
-                aiida/orm/authinfos.py|
+                .github/.*|
+                .molecule/.*|
+                docs/.*|
+                utils/.*|
+
+                aiida/calculations/arithmetic/add.py|
+                aiida/calculations/diff_tutorial/calculations.py|
+                aiida/calculations/templatereplacer.py|
+                aiida/calculations/transfer.py|
+                aiida/cmdline/commands/cmd_archive.py|
+                aiida/cmdline/commands/cmd_calcjob.py|
+                aiida/cmdline/commands/cmd_code.py|
+                aiida/cmdline/commands/cmd_computer.py|
+                aiida/cmdline/commands/cmd_data/cmd_list.py|
+                aiida/cmdline/commands/cmd_data/cmd_upf.py|
+                aiida/cmdline/commands/cmd_devel.py|
+                aiida/cmdline/commands/cmd_group.py|
+                aiida/cmdline/commands/cmd_node.py|
+                aiida/cmdline/commands/cmd_run.py|
+                aiida/cmdline/commands/cmd_shell.py|
+                aiida/cmdline/commands/cmd_storage.py|
+                aiida/cmdline/groups/dynamic.py|
+                aiida/cmdline/params/options/commands/setup.py|
+                aiida/cmdline/params/options/interactive.py|
+                aiida/cmdline/params/options/main.py|
+                aiida/cmdline/params/options/multivalue.py|
+                aiida/cmdline/params/types/group.py|
+                aiida/cmdline/params/types/plugin.py|
+                aiida/cmdline/utils/ascii_vis.py|
+                aiida/cmdline/utils/common.py|
+                aiida/cmdline/utils/echo.py|
+                aiida/common/extendeddicts.py|
+                aiida/common/hashing.py|
+                aiida/common/utils.py|
+                aiida/engine/daemon/execmanager.py|
+                aiida/engine/processes/calcjobs/manager.py|
+                aiida/engine/processes/calcjobs/monitors.py|
+                aiida/engine/processes/calcjobs/tasks.py|
+                aiida/engine/processes/control.py|
+                aiida/engine/processes/functions.py|
+                aiida/engine/processes/ports.py|
+                aiida/manage/configuration/__init__.py|
+                aiida/manage/configuration/config.py|
+                aiida/manage/configuration/profile.py|
+                aiida/manage/configuration/settings.py|
+                aiida/manage/external/rmq/launcher.py|
+                aiida/manage/tests/main.py|
+                aiida/manage/tests/pytest_fixtures.py|
                 aiida/orm/comments.py|
                 aiida/orm/computers.py|
-                aiida/orm/groups.py|
-                aiida/orm/logs.py|
-                aiida/orm/users.py|
-                aiida/orm/nodes/data/enum.py|
-                aiida/orm/nodes/data/jsonable.py|
-                aiida/orm/nodes/attributes.py|
-                aiida/orm/nodes/node.py|
-                aiida/orm/nodes/process/.*py|
-                aiida/orm/nodes/repository.py|
-                aiida/orm/utils/links.py|
-                aiida/orm/utils/mixins.py|
-                aiida/plugins/entry_point.py|
-                aiida/plugins/factories.py|
-                aiida/plugins/utils.py|
-                aiida/repository/.*py|
-                aiida/storage/control.py|
-                aiida/storage/psql_dos/backend.py|
-                aiida/storage/psql_dos/orm/querybuilder/.*py|
-                aiida/storage/psql_dos/utils.py|
-                aiida/storage/sqlite_zip/.*.py|
+                aiida/orm/implementation/storage_backend.py|
+                aiida/orm/nodes/caching.py|
+                aiida/orm/nodes/comments.py|
+                aiida/orm/nodes/data/array/array.py|
+                aiida/orm/nodes/data/array/bands.py|
+                aiida/orm/nodes/data/array/trajectory.py|
+                aiida/orm/nodes/data/cif.py|
+                aiida/orm/nodes/data/code/installed.py|
+                aiida/orm/nodes/data/code/legacy.py|
+                aiida/orm/nodes/data/code/portable.py|
+                aiida/orm/nodes/data/remote/base.py|
+                aiida/orm/nodes/data/structure.py|
+                aiida/orm/nodes/data/upf.py|
+                aiida/orm/nodes/process/calculation/calcjob.py|
+                aiida/orm/nodes/process/process.py|
+                aiida/orm/utils/builders/code.py|
+                aiida/orm/utils/builders/computer.py|
+                aiida/orm/utils/calcjob.py|
+                aiida/orm/utils/node.py|
+                aiida/orm/utils/remote.py|
+                aiida/orm/utils/serialize.py|
+                aiida/repository/backend/disk_object_store.py|
+                aiida/repository/backend/sandbox.py|
+                aiida/restapi/common/utils.py|
+                aiida/restapi/resources.py|
+                aiida/restapi/run_api.py|
+                aiida/restapi/translator/base.py|
+                aiida/restapi/translator/computer.py|
+                aiida/restapi/translator/group.py|
+                aiida/restapi/translator/nodes/.*|
+                aiida/restapi/translator/user.py|
+                aiida/schedulers/plugins/direct.py|
+                aiida/schedulers/plugins/lsf.py|
+                aiida/schedulers/plugins/pbsbaseclasses.py|
+                aiida/schedulers/plugins/sge.py|
+                aiida/schedulers/plugins/slurm.py|
+                aiida/storage/psql_dos/migrations/utils/integrity.py|
+                aiida/storage/psql_dos/migrations/utils/legacy_workflows.py|
+                aiida/storage/psql_dos/migrations/utils/migrate_repository.py|
+                aiida/storage/psql_dos/migrations/utils/parity.py|
+                aiida/storage/psql_dos/migrations/utils/reflect.py|
+                aiida/storage/psql_dos/migrations/utils/utils.py|
+                aiida/storage/psql_dos/migrations/versions/1de112340b16_django_parity_1.py|
+                aiida/storage/psql_dos/migrator.py|
+                aiida/storage/psql_dos/models/.*|
+                aiida/storage/psql_dos/orm/.*|
+                aiida/storage/sqlite_temp/backend.py|
+                aiida/storage/sqlite_zip/backend.py|
+                aiida/storage/sqlite_zip/migrations/legacy_to_main.py|
+                aiida/storage/sqlite_zip/migrator.py|
+                aiida/storage/sqlite_zip/models.py|
+                aiida/storage/sqlite_zip/orm.py|
+                aiida/tools/data/array/kpoints/legacy.py|
+                aiida/tools/data/array/kpoints/seekpath.py|
+                aiida/tools/data/orbital/orbital.py|
+                aiida/tools/data/orbital/realhydrogen.py|
+                aiida/tools/dbimporters/plugins/.*|
+                aiida/tools/graph/age_entities.py|
+                aiida/tools/graph/age_rules.py|
+                aiida/tools/graph/deletions.py|
                 aiida/tools/graph/graph_traversers.py|
                 aiida/tools/groups/paths.py|
-                aiida/tools/query/*.py|
-                aiida/tools/archive/.*py|
+                aiida/tools/query/calculation.py|
+                aiida/tools/query/mapping.py|
+                aiida/tools/visualization/graph.py|
+                aiida/transports/cli.py|
+                aiida/transports/plugins/local.py|
+                aiida/transports/plugins/ssh.py|
+                aiida/workflows/arithmetic/multiply_add.py|
+
+                tests/conftest.py|
+                tests/repository/conftest.py|
+                tests/repository/test_repository.py|
+                tests/sphinxext/sources/workchain/conf.py|
+                tests/sphinxext/sources/workchain_broken/conf.py|
+                tests/storage/psql_dos/migrations/conftest.py|
+                tests/storage/psql_dos/migrations/django_branch/test_0026_0027_traj_data.py|
+                tests/test_calculation_node.py|
+                tests/test_nodes.py|
+
             )$
 
     -   id: pylint

aiida/calculations/monitors/base.py

@@ -25,3 +25,5 @@ def always_kill(node: CalcJobNode, transport: Transport) -> str | None:  # pylin
 
     if output:
         return 'Detected a non-empty submission script'
+
+    return None

aiida/cmdline/commands/cmd_config.py

@@ -197,7 +197,7 @@ def verdi_config_caching(disabled):
 
 
 @verdi_config.command('downgrade')
-@click.argument('version', type=click.Choice({str(m.down_revision) for m in MIGRATIONS}))
+@click.argument('version', type=click.Choice(list({str(m.down_revision) for m in MIGRATIONS})))
 def verdi_config_downgrade(version):
     """Print a configuration, downgraded to a specific version."""
     path = Path(get_config_path())

aiida/cmdline/commands/cmd_data/cmd_cif.py

@@ -124,4 +124,4 @@ def cif_import(filename):
         node, _ = CifData.get_or_create(filename)
         echo.echo_success(f'imported {str(node)}')
     except ValueError as err:
-        echo.echo_critical(err)
+        echo.echo_critical(str(err))

aiida/cmdline/commands/cmd_plugin.py

@@ -23,7 +23,7 @@ def verdi_plugin():
 
 
 @verdi_plugin.command('list')
-@click.argument('entry_point_group', type=click.Choice(ENTRY_POINT_GROUP_TO_MODULE_PATH_MAP.keys()), required=False)
+@click.argument('entry_point_group', type=click.Choice(list(ENTRY_POINT_GROUP_TO_MODULE_PATH_MAP)), required=False)
 @click.argument('entry_point', type=click.STRING, required=False)
 @decorators.with_dbenv()
 def plugin_list(entry_point_group, entry_point):

aiida/cmdline/commands/cmd_process.py

@@ -149,7 +149,7 @@ def process_call_root(processes):
 @click.option(
     '-l',
     '--levelname',
-    type=click.Choice(LOG_LEVELS.keys()),
+    type=click.Choice(list(LOG_LEVELS)),
     default='REPORT',
     help='Filter the results by name of the log level.'
 )

aiida/cmdline/commands/cmd_rabbitmq.py

@@ -245,7 +245,7 @@ def get_active_processes() -> list[int]:
     from aiida.engine import ProcessState
     from aiida.orm import ProcessNode, QueryBuilder
 
-    return QueryBuilder().append(
+    return QueryBuilder().append(  # type: ignore[return-value]
         ProcessNode,
         filters={
             'attributes.process_state': {

aiida/cmdline/utils/decorators.py

@@ -199,7 +199,7 @@ def do_circus_stuff():
     """
     import zmq
     try:
-        zmq_version = [int(part) for part in zmq.__version__.split('.')[:2]]
+        zmq_version = [int(part) for part in zmq.__version__.split('.')[:2]]  # type: ignore
         if len(zmq_version) < 2:
             raise ValueError()
     except (AttributeError, ValueError):

aiida/cmdline/utils/echo.py

@@ -12,6 +12,7 @@
 import enum
 import json
 import sys
+from typing import Any, Optional
 
 import click
 import yaml
@@ -44,7 +45,7 @@ class ExitCode(enum.IntEnum):
 }
 
 
-def echo(message: str, fg: str = None, bold: bool = False, nl: bool = True, err: bool = False) -> None:
+def echo(message: Any, fg: Optional[str] = None, bold: bool = False, nl: bool = True, err: bool = False) -> None:
     """Log a message to the cmdline logger.
 
     .. note:: The message will be logged at the ``REPORT`` level but always without the log level prefix.

aiida/cmdline/utils/pluginable.py

@@ -8,7 +8,7 @@
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
 """Plugin aware click command Group."""
-from aiida.cmdline.commands.cmd_verdi import VerdiCommandGroup
+from aiida.cmdline.groups import VerdiCommandGroup
 from aiida.common import exceptions
 from aiida.plugins.entry_point import get_entry_point_names, load_entry_point
 

aiida/common/lang.py

@@ -11,7 +11,7 @@
 import functools
 import inspect
 import keyword
-from typing import Any, Callable, Generic, Type, TypeVar
+from typing import Any, Callable, Generic, TypeVar
 
 
 def isidentifier(identifier):
@@ -45,10 +45,13 @@ def type_check(what, of_type, msg=None, allow_none=False):
     return what
 
 
-def override_decorator(check=False):
+MethodType = TypeVar('MethodType', bound=Callable[..., Any])
+
+
+def override_decorator(check=False) -> Callable[[MethodType], MethodType]:
     """Decorator to signal that a method from a base class is being overridden completely."""
 
-    def wrap(func):  # pylint: disable=missing-docstring
+    def wrap(func: MethodType) -> MethodType:  # pylint: disable=missing-docstring
         if isinstance(func, property):
             raise RuntimeError('Override must go after @property decorator')
 
@@ -69,7 +72,7 @@ def wrapped_fn(self, *args, **kwargs):  # pylint: disable=missing-docstring
         else:
             wrapped_fn = func
 
-        return wrapped_fn
+        return wrapped_fn  # type: ignore
 
     return wrap
 
@@ -89,8 +92,8 @@ class classproperty(Generic[ReturnType]):  # pylint: disable=invalid-name
     instance as its first argument).
     """
 
-    def __init__(self, getter: Callable[[Type[SelfType]], ReturnType]) -> None:
+    def __init__(self, getter: Callable[[SelfType], ReturnType]) -> None:
         self.getter = getter
 
-    def __get__(self, instance: Any, owner: Type[SelfType]) -> ReturnType:
+    def __get__(self, instance: Any, owner: SelfType) -> ReturnType:
         return self.getter(owner)

aiida/common/log.py

@@ -12,6 +12,7 @@
 import contextlib
 import logging
 import types
+from typing import cast
 
 __all__ = ('AIIDA_LOGGER', 'override_log_level')
 
@@ -24,11 +25,17 @@
 logging.addLevelName(LOG_LEVEL_REPORT, 'REPORT')
 
 
-def report(self, msg, *args, **kwargs):
+def report(self: logging.Logger, msg, *args, **kwargs):
     """Log a message at the ``REPORT`` level."""
     self.log(LOG_LEVEL_REPORT, msg, *args, **kwargs)
 
 
+class AiidaLoggerType(logging.Logger):
+
+    def report(self, msg: str, *args, **kwargs) -> None:
+        """Log a message at the ``REPORT`` level."""
+
+
 setattr(logging, 'REPORT', LOG_LEVEL_REPORT)
 setattr(logging.Logger, 'report', report)
 
@@ -43,7 +50,7 @@ def report(self, msg, *args, **kwargs):
     logging.getLevelName(logging.CRITICAL): logging.CRITICAL,
 }
 
-AIIDA_LOGGER = logging.getLogger('aiida')
+AIIDA_LOGGER = cast(AiidaLoggerType, logging.getLogger('aiida'))
 CLI_LOG_LEVEL = None
 
 

aiida/common/timezone.py

@@ -9,6 +9,7 @@
 ###########################################################################
 """Utility functions to operate on datetime objects."""
 from datetime import datetime, timedelta, timezone, tzinfo
+from typing import Optional
 
 import pytz
 
@@ -23,7 +24,7 @@ def now() -> datetime:
     return datetime.now().astimezone()
 
 
-def make_aware(value: datetime, tz: tzinfo = None) -> datetime:
+def make_aware(value: datetime, tz: Optional[tzinfo] = None) -> datetime:
     """Make the given datetime object timezone aware.
 
     :param value: The datetime object to make aware.
@@ -56,7 +57,7 @@ def timezone_from_name(name: str) -> tzinfo:
         raise ValueError(f'unknown timezone: {name}') from exception
 
 
-def delta(from_time: datetime, to_time: datetime = None) -> timedelta:
+def delta(from_time: datetime, to_time: Optional[datetime] = None) -> timedelta:
     """Return the datetime object representing the different between two datetime objects.
 
     :param from_time: The starting datetime object.

aiida/engine/launch.py

@@ -14,8 +14,9 @@
 from aiida.manage import manager
 from aiida.orm import ProcessNode
 
+from .processes.builder import ProcessBuilder
 from .processes.functions import FunctionProcess
-from .processes.process import Process, ProcessBuilder
+from .processes.process import Process
 from .utils import instantiate_process, is_process_scoped  # pylint: disable=no-name-in-module
 
 __all__ = ('run', 'run_get_pk', 'run_get_node', 'submit')