add support for forms, values, values_list

Author: mkurnikov

django-stubs/apps/registry.pyi

@@ -23,8 +23,9 @@ class Apps:
     def check_models_ready(self) -> None: ...
     def get_app_configs(self) -> Iterable[AppConfig]: ...
     def get_app_config(self, app_label: str) -> AppConfig: ...
-    def get_models(self, include_auto_created: bool = ..., include_swapped: bool = ...) -> List[Type[Model]]: ...
-    def get_model(self, app_label: str, model_name: Optional[str] = ..., require_ready: bool = ...) -> Type[Model]: ...
+    # it's not possible to support it in plugin properly now
+    def get_models(self, include_auto_created: bool = ..., include_swapped: bool = ...) -> List[Type[Any]]: ...
+    def get_model(self, app_label: str, model_name: Optional[str] = ..., require_ready: bool = ...) -> Type[Any]: ...
     def register_model(self, app_label: str, model: Type[Model]) -> None: ...
     def is_installed(self, app_name: str) -> bool: ...
     def get_containing_app_config(self, object_name: str) -> Optional[AppConfig]: ...

django-stubs/db/models/sql/query.pyi

@@ -100,6 +100,7 @@ class Query:
     def resolve_expression(self, query: Query, *args: Any, **kwargs: Any) -> Query: ...
     def as_sql(self, compiler: SQLCompiler, connection: Any) -> Tuple[str, Tuple]: ...
     def resolve_lookup_value(self, value: Any, can_reuse: Optional[Set[str]], allow_joins: bool) -> Any: ...
+    def solve_lookup_type(self, lookup: str) -> Tuple[Sequence[str], Sequence[str], bool]: ...
     def build_filter(
         self,
         filter_expr: Union[Dict[str, str], Tuple[str, Tuple[int, int]]],

mypy_django_plugin_newsemanal/django/context.py

@@ -1,10 +1,11 @@
 import os
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import Any, Dict, Iterator, List, Optional, TYPE_CHECKING, Tuple, Type
+from typing import Any, Dict, Iterator, List, Optional, TYPE_CHECKING, Tuple, Type, Sequence
 
+from django.core.exceptions import FieldError
 from django.db.models.base import Model
-from django.db.models.fields.related import ForeignKey
+from django.db.models.fields.related import ForeignKey, RelatedField
 from django.utils.functional import cached_property
 from mypy.checker import TypeChecker
 from mypy.types import Instance, Type as MypyType
@@ -13,6 +14,8 @@
 from django.contrib.postgres.fields import ArrayField
 from django.db.models.fields import CharField, Field
 from django.db.models.fields.reverse_related import ForeignObjectRel
+
+from django.db.models.sql.query import Query
 from mypy_django_plugin_newsemanal.lib import helpers
 
 if TYPE_CHECKING:
@@ -53,6 +56,9 @@ def noop_class_getitem(cls, key):
 
 
 class DjangoFieldsContext:
+    def __init__(self, django_context: 'DjangoContext') -> None:
+        self.django_context = django_context
+
     def get_attname(self, field: Field) -> str:
         attname = field.attname
         return attname
@@ -81,11 +87,43 @@ def get_field_set_type(self, api: TypeChecker, field: Field, method: str) -> Myp
             field_set_type = helpers.convert_any_to_type(field_set_type, argument_field_type)
         return field_set_type
 
+    def get_field_get_type(self, api: TypeChecker, field: Field, method: str) -> MypyType:
+        field_info = helpers.lookup_class_typeinfo(api, field.__class__)
+        is_nullable = self.get_field_nullability(field, method)
+        if isinstance(field, RelatedField):
+            if method == 'values':
+                primary_key_field = self.django_context.get_primary_key_field(field.related_model)
+                return self.get_field_get_type(api, primary_key_field, method)
+
+            model_info = helpers.lookup_class_typeinfo(api, field.related_model)
+            return Instance(model_info, [])
+        else:
+            return helpers.get_private_descriptor_type(field_info, '_pyi_private_get_type',
+                                                                 is_nullable=is_nullable)
+
+
+class DjangoLookupsContext:
+    def resolve_lookup(self, model_cls: Type[Model], lookup: str) -> Any:
+        query = Query(model_cls)
+        lookup_parts, field_parts, is_expression = query.solve_lookup_type(lookup)
+        if lookup_parts:
+            raise FieldError('Lookups not supported yet')
+
+        currently_observed_model = model_cls
+        current_field = None
+        for field_name in field_parts:
+            current_field = currently_observed_model._meta.get_field(field_name)
+            if isinstance(current_field, RelatedField):
+                currently_observed_model = current_field.related_model
+
+        return current_field
+
 
 class DjangoContext:
     def __init__(self, plugin_toml_config: Optional[Dict[str, Any]]) -> None:
         self.config = DjangoPluginConfig()
-        self.fields_context = DjangoFieldsContext()
+        self.fields_context = DjangoFieldsContext(self)
+        self.lookups_context = DjangoLookupsContext()
 
         self.django_settings_module = None
         if plugin_toml_config:

mypy_django_plugin_newsemanal/lib/helpers.py

@@ -1,9 +1,17 @@
-from typing import Dict, List, Optional, Set, Union
+from collections import OrderedDict
+from typing import Dict, List, Optional, Set, Union, Any
 
+from mypy import checker
 from mypy.checker import TypeChecker
-from mypy.nodes import Expression, MypyFile, NameExpr, SymbolNode, TypeInfo, Var, SymbolTableNode
-from mypy.plugin import FunctionContext, MethodContext
-from mypy.types import AnyType, Instance, NoneTyp, Type as MypyType, TypeOfAny, UnionType
+from mypy.mro import calculate_mro
+from mypy.nodes import Block, ClassDef, Expression, GDEF, MDEF, MypyFile, NameExpr, SymbolNode, SymbolTable, SymbolTableNode, \
+    TypeInfo, Var
+from mypy.plugin import CheckerPluginInterface, FunctionContext, MethodContext
+from mypy.types import AnyType, Instance, NoneTyp, TupleType, Type as MypyType, TypeOfAny, TypedDictType, UnionType
+
+
+def get_django_metadata(model_info: TypeInfo) -> Dict[str, Any]:
+    return model_info.metadata.setdefault('django', {})
 
 
 class IncompleteDefnException(Exception):
@@ -120,6 +128,53 @@ def get_nested_meta_node_for_current_class(info: TypeInfo) -> Optional[TypeInfo]
     return None
 
 
+def add_new_class_for_current_module(api: TypeChecker, name: str, bases: List[Instance],
+                                     fields: 'OrderedDict[str, MypyType]') -> TypeInfo:
+    current_module = api.scope.stack[0]
+    new_class_unique_name = checker.gen_unique_name(name, current_module.names)
+
+    # make new class expression
+    classdef = ClassDef(new_class_unique_name, Block([]))
+    classdef.fullname = current_module.fullname() + '.' + new_class_unique_name
+
+    # make new TypeInfo
+    new_typeinfo = TypeInfo(SymbolTable(), classdef, current_module.fullname())
+    new_typeinfo.bases = bases
+    calculate_mro(new_typeinfo)
+    new_typeinfo.calculate_metaclass_type()
+
+    def add_field_to_new_typeinfo(var: Var, is_initialized_in_class: bool = False,
+                                  is_property: bool = False) -> None:
+        var.info = new_typeinfo
+        var.is_initialized_in_class = is_initialized_in_class
+        var.is_property = is_property
+        var._fullname = new_typeinfo.fullname() + '.' + var.name()
+        new_typeinfo.names[var.name()] = SymbolTableNode(MDEF, var)
+
+    # add fields
+    var_items = [Var(item, typ) for item, typ in fields.items()]
+    for var_item in var_items:
+        add_field_to_new_typeinfo(var_item, is_property=True)
+
+    classdef.info = new_typeinfo
+    current_module.names[new_class_unique_name] = SymbolTableNode(GDEF, new_typeinfo, plugin_generated=True)
+    return new_typeinfo
+
+
+def make_oneoff_named_tuple(api: TypeChecker, name: str, fields: 'OrderedDict[str, MypyType]') -> TupleType:
+    namedtuple_info = add_new_class_for_current_module(api, name,
+                                                       bases=[api.named_generic_type('typing.NamedTuple', [])],
+                                                       fields=fields)
+    return TupleType(list(fields.values()), fallback=Instance(namedtuple_info, []))
+
+
+def make_tuple(api: 'TypeChecker', fields: List[MypyType]) -> TupleType:
+    # fallback for tuples is any builtins.tuple instance
+    fallback = api.named_generic_type('builtins.tuple',
+                                      [AnyType(TypeOfAny.special_form)])
+    return TupleType(fields, fallback=fallback)
+
+
 def convert_any_to_type(typ: MypyType, referred_to_type: MypyType) -> MypyType:
     if isinstance(typ, UnionType):
         converted_items = []
@@ -140,3 +195,10 @@ def convert_any_to_type(typ: MypyType, referred_to_type: MypyType) -> MypyType:
         return referred_to_type
 
     return typ
+
+
+def make_typeddict(api: CheckerPluginInterface, fields: 'OrderedDict[str, MypyType]',
+                   required_keys: Set[str]) -> TypedDictType:
+    object_type = api.named_generic_type('mypy_extensions._TypedDict', [])
+    typed_dict_type = TypedDictType(fields, required_keys=required_keys, fallback=object_type)
+    return typed_dict_type

mypy_django_plugin_newsemanal/lib/metadata.py

@@ -1,17 +1,17 @@
 import os
 from functools import partial
-from typing import Callable, Dict, List, Optional, Tuple, Type
+from typing import Callable, Dict, List, Optional, Tuple
 
 import toml
+from django.db.models.fields.related import RelatedField
 from mypy.nodes import MypyFile, TypeInfo
 from mypy.options import Options
-from mypy.plugin import ClassDefContext, FunctionContext, Plugin, MethodContext, AttributeContext
+from mypy.plugin import AttributeContext, ClassDefContext, FunctionContext, MethodContext, Plugin
 from mypy.types import Type as MypyType
 
-from django.db.models.fields.related import RelatedField
 from mypy_django_plugin_newsemanal.django.context import DjangoContext
-from mypy_django_plugin_newsemanal.lib import fullnames, metadata
-from mypy_django_plugin_newsemanal.transformers import fields, settings, querysets, init_create
+from mypy_django_plugin_newsemanal.lib import fullnames, helpers
+from mypy_django_plugin_newsemanal.transformers import fields, forms, init_create, querysets, settings
 from mypy_django_plugin_newsemanal.transformers.models import process_model_class
 
 
@@ -20,7 +20,7 @@ def transform_model_class(ctx: ClassDefContext,
     sym = ctx.api.lookup_fully_qualified_or_none(fullnames.MODEL_CLASS_FULLNAME)
 
     if sym is not None and isinstance(sym.node, TypeInfo):
-        metadata.get_django_metadata(sym.node)['model_bases'][ctx.cls.fullname] = 1
+        helpers.get_django_metadata(sym.node)['model_bases'][ctx.cls.fullname] = 1
     else:
         if not ctx.api.final_iteration:
             ctx.api.defer()
@@ -29,10 +29,18 @@ def transform_model_class(ctx: ClassDefContext,
     process_model_class(ctx, django_context)
 
 
+def transform_form_class(ctx: ClassDefContext) -> None:
+    sym = ctx.api.lookup_fully_qualified_or_none(fullnames.BASEFORM_CLASS_FULLNAME)
+    if sym is not None and isinstance(sym.node, TypeInfo):
+        helpers.get_django_metadata(sym.node)['baseform_bases'][ctx.cls.fullname] = 1
+
+    forms.make_meta_nested_class_inherit_from_any(ctx)
+
+
 def add_new_manager_base(ctx: ClassDefContext) -> None:
     sym = ctx.api.lookup_fully_qualified_or_none(fullnames.MANAGER_CLASS_FULLNAME)
     if sym is not None and isinstance(sym.node, TypeInfo):
-        metadata.get_django_metadata(sym.node)['manager_bases'][ctx.cls.fullname] = 1
+        helpers.get_django_metadata(sym.node)['manager_bases'][ctx.cls.fullname] = 1
 
 
 class NewSemanalDjangoPlugin(Plugin):
@@ -50,24 +58,34 @@ def __init__(self, options: Options) -> None:
     def _get_current_queryset_bases(self) -> Dict[str, int]:
         model_sym = self.lookup_fully_qualified(fullnames.QUERYSET_CLASS_FULLNAME)
         if model_sym is not None and isinstance(model_sym.node, TypeInfo):
-            return (metadata.get_django_metadata(model_sym.node)
+            return (helpers.get_django_metadata(model_sym.node)
                     .setdefault('queryset_bases', {fullnames.QUERYSET_CLASS_FULLNAME: 1}))
         else:
             return {}
 
     def _get_current_manager_bases(self) -> Dict[str, int]:
         model_sym = self.lookup_fully_qualified(fullnames.MANAGER_CLASS_FULLNAME)
         if model_sym is not None and isinstance(model_sym.node, TypeInfo):
-            return (metadata.get_django_metadata(model_sym.node)
+            return (helpers.get_django_metadata(model_sym.node)
                     .setdefault('manager_bases', {fullnames.MANAGER_CLASS_FULLNAME: 1}))
         else:
             return {}
 
     def _get_current_model_bases(self) -> Dict[str, int]:
         model_sym = self.lookup_fully_qualified(fullnames.MODEL_CLASS_FULLNAME)
         if model_sym is not None and isinstance(model_sym.node, TypeInfo):
-            return metadata.get_django_metadata(model_sym.node).setdefault('model_bases',
-                                                                           {fullnames.MODEL_CLASS_FULLNAME: 1})
+            return helpers.get_django_metadata(model_sym.node).setdefault('model_bases',
+                                                                          {fullnames.MODEL_CLASS_FULLNAME: 1})
+        else:
+            return {}
+
+    def _get_current_form_bases(self) -> Dict[str, int]:
+        model_sym = self.lookup_fully_qualified(fullnames.BASEFORM_CLASS_FULLNAME)
+        if model_sym is not None and isinstance(model_sym.node, TypeInfo):
+            return (helpers.get_django_metadata(model_sym.node)
+                    .setdefault('baseform_bases', {fullnames.BASEFORM_CLASS_FULLNAME: 1,
+                                                   fullnames.FORM_CLASS_FULLNAME: 1,
+                                                   fullnames.MODELFORM_CLASS_FULLNAME: 1}))
         else:
             return {}
 
@@ -85,15 +103,20 @@ def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]:
         if file.fullname() == 'django.conf' and self.django_context.django_settings_module:
             return [self._new_dependency(self.django_context.django_settings_module)]
 
+        # for values / values_list
+        if file.fullname() == 'django.db.models':
+            return [self._new_dependency('mypy_extensions'), self._new_dependency('typing')]
+
         # for `get_user_model()`
-        if file.fullname() == 'django.contrib.auth':
-            auth_user_model_name = self.django_context.settings.AUTH_USER_MODEL
-            try:
-                auth_user_module = self.django_context.apps_registry.get_model(auth_user_model_name).__module__
-            except LookupError:
-                # get_user_model() model app is not installed
-                return []
-            return [self._new_dependency(auth_user_module)]
+        if self.django_context.settings:
+            if file.fullname() == 'django.contrib.auth':
+                auth_user_model_name = self.django_context.settings.AUTH_USER_MODEL
+                try:
+                    auth_user_module = self.django_context.apps_registry.get_model(auth_user_model_name).__module__
+                except LookupError:
+                    # get_user_model() model app is not installed
+                    return []
+                return [self._new_dependency(auth_user_module)]
 
         # ensure that all mentioned to='someapp.SomeModel' are loaded with corresponding related Fields
         defined_model_classes = self.django_context.model_modules.get(file.fullname())
@@ -132,9 +155,29 @@ def get_function_hook(self, fullname: str
                 return partial(init_create.redefine_and_typecheck_model_init, django_context=self.django_context)
 
     def get_method_hook(self, fullname: str
-                        ) -> Optional[Callable[[MethodContext], Type]]:
-        manager_classes = self._get_current_manager_bases()
+                        ) -> Optional[Callable[[MethodContext], MypyType]]:
         class_fullname, _, method_name = fullname.rpartition('.')
+        if method_name == 'get_form_class':
+            info = self._get_typeinfo_or_none(class_fullname)
+            if info and info.has_base(fullnames.FORM_MIXIN_CLASS_FULLNAME):
+                return forms.extract_proper_type_for_get_form_class
+
+        if method_name == 'get_form':
+            info = self._get_typeinfo_or_none(class_fullname)
+            if info and info.has_base(fullnames.FORM_MIXIN_CLASS_FULLNAME):
+                return forms.extract_proper_type_for_get_form
+
+        if method_name == 'values':
+            model_info = self._get_typeinfo_or_none(class_fullname)
+            if model_info and model_info.has_base(fullnames.QUERYSET_CLASS_FULLNAME):
+                return partial(querysets.extract_proper_type_queryset_values, django_context=self.django_context)
+
+        if method_name == 'values_list':
+            model_info = self._get_typeinfo_or_none(class_fullname)
+            if model_info and model_info.has_base(fullnames.QUERYSET_CLASS_FULLNAME):
+                return partial(querysets.extract_proper_type_queryset_values_list, django_context=self.django_context)
+
+        manager_classes = self._get_current_manager_bases()
         if class_fullname in manager_classes and method_name == 'create':
             return partial(init_create.redefine_and_typecheck_model_create, django_context=self.django_context)
 
@@ -146,19 +189,16 @@ def get_base_class_hook(self, fullname: str
         if fullname in self._get_current_manager_bases():
             return add_new_manager_base
 
+        if fullname in self._get_current_form_bases():
+            return transform_form_class
+
     def get_attribute_hook(self, fullname: str
                            ) -> Optional[Callable[[AttributeContext], MypyType]]:
         class_name, _, attr_name = fullname.rpartition('.')
         if class_name == fullnames.DUMMY_SETTINGS_BASE_CLASS:
             return partial(settings.get_type_of_settings_attribute,
                            django_context=self.django_context)
 
-    # def get_type_analyze_hook(self, fullname: str
-    #                           ) -> Optional[Callable[[AnalyzeTypeContext], MypyType]]:
-    #     queryset_bases = self._get_current_queryset_bases()
-    #     if fullname in queryset_bases:
-    #         return partial(querysets.set_first_generic_param_as_default_for_second, fullname=fullname)
-
 
 def plugin(version):
     return NewSemanalDjangoPlugin