Delay finding names of arg construcors

Author: sixolet

extensions/mypy_extensions.py

@@ -98,20 +98,26 @@ class Point2D(TypedDict):
 # Return type that indicates a function does not return
 class NoReturn: pass
 
-def Arg(name=None, typ=Any):
+
+def Arg(typ=Any, name=None):
     return typ
 
-def DefaultArg(name=None, typ=Any):
+
+def DefaultArg(typ=Any, name=None):
     return typ
 
-def NamedArg(name=None, typ=Any):
+
+def NamedArg(typ=Any, name=None):
     return typ
 
-def DefaultNamedArg(name=None, typ=Any):
+
+def DefaultNamedArg(typ=Any, name=None):
     return typ
 
+
 def StarArg(typ=Any):
     return typ
 
+
 def KwArg(typ=Any):
     return typ

mypy/exprtotype.py

@@ -56,19 +56,18 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type:
     elif isinstance(expr, ListExpr):
         types = []  # type: List[Type]
         names = []  # type: List[Optional[str]]
-        kinds = []  # type: List[int]
+        constructors = []  # type: List[UnboundArgumentConstructor]
         for it in expr.items:
             if isinstance(it, CallExpr):
-                if not isinstance(it.callee, NameExpr):
-                    raise TypeTranslationError()
-                arg_const = it.callee.name
-                try:
-                    kind = ARG_KINDS_BY_CONSTRUCTOR[arg_const]
-                except KeyError:
+                callee = it.callee
+                if isinstance(callee, NameExpr):
+                    constructor = UnboundArgumentConstructor(callee.name)
+                elif isinstance(it.callee, MemberExpr):
+                    constructor = UnboundArgumentConstructor(get_member_expr_fullname(callee))
+                else:
                     raise TypeTranslationError()
                 name = None
                 typ = AnyType(implicit=True)  # type: Type
-                star = arg_const in STAR_ARG_CONSTRUCTORS
                 for i, arg in enumerate(it.args):
                     if it.arg_names[i] is not None:
                         if it.arg_names[i] == "name":
@@ -79,21 +78,21 @@ def expr_to_unanalyzed_type(expr: Expression) -> Type:
                             continue
                         else:
                             raise TypeTranslationError()
-                    elif i == 0 and not star:
-                        name = _extract_str(arg)
-                    elif i == 1 and not star or i == 0 and star:
+                    elif i == 0:
                         typ = expr_to_unanalyzed_type(arg)
+                    elif i == 1:
+                        name = _extract_str(arg)
                     else:
                         raise TypeTranslationError()
                 names.append(name)
                 types.append(typ)
-                kinds.append(kind)
+                constructors.append(constructor)
             else:
                 types.append(expr_to_unanalyzed_type(it))
                 names.append(None)
-                kinds.append(ARG_POS)
-        return ArgumentList(types, names, kinds,
-                        line=expr.line, column=expr.column)
+                constructors.append(UnboundArgumentConstructor(None))
+        return ArgumentList(types, names, constructors,
+                            line=expr.line, column=expr.column)
     elif isinstance(expr, (StrExpr, BytesExpr, UnicodeExpr)):
         # Parse string literal type.
         try:

mypy/fastparse.py

@@ -953,59 +953,51 @@ def translate_expr_list(self, l: Sequence[ast3.AST]) -> List[Type]:
     def translate_argument_list(self, l: Sequence[ast3.AST]) -> ArgumentList:
         types = []  # type: List[Type]
         names = []  # type: List[Optional[str]]
-        kinds = []  # type: List[int]
+        constructors = []  # type: List[UnboundArgumentConstructor]
         for e in l:
+            constructor = UnboundArgumentConstructor(None)
             if isinstance(e, ast3.Call):
+                name = None  # type: Optional[str]
                 # Parse the arg constructor
                 f = e.func
-                if not isinstance(f, ast3.Name):
-                    raise FastParserError("Expected arg constructor name",
-                                          e.lineno, e.col_offset)
-                try:
-                    kind = ARG_KINDS_BY_CONSTRUCTOR[f.id]
-                    star = f.id in STAR_ARG_CONSTRUCTORS
-                except KeyError:
-                    raise FastParserError("Unknown argument constructor {}".format(f.id),
-                                          f.lineno, f.col_offset)
+                if isinstance(f, ast3.Name):
+                    constructor = UnboundArgumentConstructor(f.id, line=self.line)
+                elif isinstance(f, ast3.NameConstant):
+                    constructor = UnboundArgumentConstructor(str(f.value), line=self.line)
+                elif isinstance(f, ast3.Attribute):
+                    before_dot = self.visit(f.value)
+                    if isinstance(before_dot, UnboundType):
+                        constructor = UnboundArgumentConstructor(
+                            "{}.{}".format(before_dot.name, f.attr), line = self.line)
 
-                name = None  # type: Optional[str]
                 typ = AnyType(implicit=True)  # type: Type
-                for i, arg in enumerate(e.args):
-                    if i == 0 and not star:
-                        name = _extract_str(arg)
-                    elif i == 1 and not star or i == 0 and star:
-                        try:
-                            typ = self.visit(arg)
-                        except TypeCommentParseError:
-                            raise FastParserError("Bad type for callable argument",
-                                                  arg.lineno, arg.col_offset)
-                    else:
-                        raise FastParserError("Too many arguments for argument constructor",
-                                              f.lineno, f.col_offset)
+                if len(e.args) >= 1:
+                    typ = self.visit(e.args[0])
+                if len(e.args) >= 2:
+                    name = self._extract_str(e.args[1])
+                if len(e.args) > 2:
+                    self.fail("Too many arguments for argument constructor",
+                              f.lineno, getattr(f, 'col_offset', -1))
                 for k in e.keywords:
                     value = k.value
                     if k.arg == "name" and not star:
-                        name = _extract_str(value)
+                        name = self._extract_str(value)
                     elif k.arg == "typ":
-                        try:
-                            typ = self.visit(value)
-                        except TypeCommentParseError:
-                            raise FastParserError("Bad type for callable argument",
-                                                  value.lineno, value.col_offset)
+                        typ = self.visit(value)
                     else:
-                        raise FastParserError(
+                        self.fail(
                             'Unexpected argument "{}" for argument constructor'.format(k.arg),
-                            value.lineno, value.col_offset)
+                            value.lineno, getattr(value, 'col_offset', -1))
 
                 types.append(typ)
                 names.append(name)
-                kinds.append(kind)
+                constructors.append(constructor)
             else:
                 types.append(self.visit(e))
                 names.append(None)
-                kinds.append(ARG_POS)
+                constructors.append(constructor)
 
-        return ArgumentList(types, names, kinds, line=self.line)
+        return ArgumentList(types, names, constructors, line=self.line)
 
     def visit_Name(self, n: ast3.Name) -> Type:
         return UnboundType(n.id, line=self.line)
@@ -1060,25 +1052,12 @@ def visit_Ellipsis(self, n: ast3.Ellipsis) -> Type:
     def visit_List(self, n: ast3.List) -> Type:
         return self.translate_argument_list(n.elts)
 
-
-class TypeCommentParseError(Exception):
-    def __init__(self, msg: str, lineno: int, offset: int) -> None:
-        self.msg = msg
-        self.lineno = lineno
-        self.offset = offset
-
-
-class FastParserError(TypeCommentParseError):
-    pass
-
-
-def _extract_str(arg: ast3.expr) -> Optional[str]:
-    if isinstance(arg, ast3.Name) and arg.id == 'None':
-        return None
-    elif isinstance(arg, ast3.NameConstant) and arg.value is None:
-        return None
-    elif isinstance(arg, ast3.Str):
-        return arg.s
-    else:
-        raise FastParserError("Bad type for name of argument",
-                              arg.lineno, arg.col_offset)
+    def _extract_str(arg: ast3.expr) -> Optional[str]:
+        if isinstance(arg, ast3.Name) and arg.id == 'None':
+            return None
+        elif isinstance(arg, ast3.NameConstant) and arg.value is None:
+            return None
+        elif isinstance(arg, ast3.Str):
+            return arg.s
+        else:
+            self.fail("Bad type for name of argument", arg.lineno, arg.col_offset)

mypy/messages.py

@@ -216,8 +216,8 @@ def format(self, typ: Type, verbosity: int = 0) -> str:
                         else:
                             arg_strings.append("{}({}, {})".format(
                                 constructor,
-                                repr(arg_name),
-                                strip_quotes(self.format(arg_type))))
+                                strip_quotes(self.format(arg_type)),
+                                repr(arg_name)))
 
                 return 'Callable[[{}], {}]'.format(", ".join(arg_strings), return_type)
             else:

mypy/semanal.py

@@ -766,8 +766,8 @@ def get_tvars(self, tp: Type) -> List[Tuple[str, TypeVarExpr]]:
         tvars = []  # type: List[Tuple[str, TypeVarExpr]]
         if isinstance(tp, UnboundType):
             tp_args = tp.args
-        elif isinstance(tp, TypeList):
-            tp_args = tp.items
+        elif isinstance(tp, ArgumentList):
+            tp_args = tp.types
         else:
             return tvars
         for arg in tp_args:

mypy/typeanal.py

@@ -29,6 +29,15 @@
     'typing.Union',
 }
 
+ARG_KINDS_BY_CONSTRUCTOR = {
+    'mypy_extensions.Arg': nodes.ARG_POS,
+    'mypy_extensions.DefaultArg': nodes.ARG_OPT,
+    'mypy_extensions.NamedArg': nodes.ARG_NAMED,
+    'mypy_extensions.DefaultNamedArg': nodes.ARG_NAMED_OPT,
+    'mypy_extensions.StarArg': nodes.ARG_STAR,
+    'mypy_extensions.KwArg': nodes.ARG_STAR2,
+}
+
 
 def analyze_type_alias(node: Expression,
                        lookup_func: Callable[[str, Context], SymbolTableNode],
@@ -330,6 +339,27 @@ def visit_ellipsis_type(self, t: EllipsisType) -> Type:
     def visit_type_type(self, t: TypeType) -> Type:
         return TypeType(t.item.accept(self), line=t.line)
 
+    def argument_list_to_kinds(self, a: ArgumentList) -> List[int]:
+        res = []  # type: List[int]
+        for constructor in a.constructors:
+            if constructor.name is None:
+                res.append(nodes.ARG_POS)
+            else:
+                sym = self.lookup(constructor.name, constructor)
+                if sym.node is None:
+                    assert sym.kind == UNBOUND_IMPORTED
+                    self.fail("Argument constructor not found: {}".format(constructor.name),
+                              constructor)
+                    res.append(nodes.ARG_POS)
+                    continue
+                fullname = sym.node.fullname()
+                if fullname not in ARG_KINDS_BY_CONSTRUCTOR:
+                    self.fail("Not an argument constructor: {}".format(fullname), constructor)
+                    res.append(nodes.ARG_POS)
+                else:
+                    res.append(ARG_KINDS_BY_CONSTRUCTOR[fullname])
+        return res
+
     def analyze_callable_type(self, t: UnboundType) -> Type:
         fallback = self.builtin_type('builtins.function')
         if len(t.args) == 0:
@@ -344,10 +374,11 @@ def analyze_callable_type(self, t: UnboundType) -> Type:
             ret_type = t.args[1].accept(self)
             if isinstance(t.args[0], ArgumentList):
                 # Callable[[ARG, ...], RET] (ordinary callable type)
+                kinds = self.argument_list_to_kinds(t.args[0])
                 args = t.args[0].types
                 try:
                     return CallableType(self.anal_array(args),
-                                        t.args[0].kinds,
+                                        kinds,
                                         t.args[0].names,
                                         ret_type=ret_type,
                                         fallback=fallback)

mypy/types.py

@@ -241,18 +241,18 @@ class ArgumentList(Type):
 
     types = None  # type: List[Type]
     names = None  # type: List[Optional[str]]
-    kinds = None  # type: List[int]
+    constructors = None  # type: List[UnboundArgumentConstructor]
 
     def __init__(self,
                  types: List[Type],
                  names: List[Optional[str]],
-                 kinds: List[int],
+                 constructors: List[UnboundArgumentConstructor],
                  line: int = -1,
                  column: int = -1) -> None:
         super().__init__(line, column)
         self.types = types
         self.names = names
-        self.kinds = kinds
+        self.constructors = constructors
 
     def accept(self, visitor: 'TypeVisitor[T]') -> T:
         return visitor.visit_type_list(self)
@@ -261,19 +261,15 @@ def serialize(self) -> JsonDict:
         return {'.class': 'ArgumentList',
                 'items': [t.serialize() for t in self.types],
                 'names': self.names,
-                'kinds': self.kinds}
+                'constructors': self.constructors}
 
     @classmethod
     def deserialize(cls, data: JsonDict) -> 'ArgumentList':
         assert data['.class'] == 'ArgumentList' or data['.class'] == 'TypeList'
         types = [Type.deserialize(t) for t in data['items']]
         names = cast(List[Optional[str]], data.get('names', [None] * len(types)))
-        kinds = cast(List[int], data.get('kinds', [ARG_POS] * len(types)))
-        return ArgumentList(
-            types=[Type.deserialize(t) for t in data['items']],
-            names=names,
-            kinds=kinds,
-        )
+        constructors = [UnboundArgumentConstructor.deserialize(c) for c in data['constructors']]
+        return ArgumentList(types=types, names=names, constructors=constructors)
 
 
 class AnyType(Type):
@@ -648,7 +644,7 @@ def __init__(self,
         self.special_sig = special_sig
         super().__init__(line, column)
 
-    def _process_names_on_init(self, arg_names):
+    def _process_names_on_init(self, arg_names: Iterable[str]) -> None:
         seen = set()  # type: Set[str]
         for name in arg_names:
             if name is None:
@@ -657,7 +653,7 @@ def _process_names_on_init(self, arg_names):
                 raise ArgNameException('Duplicate argument name "{}"'.format(name))
             seen.add(name)
 
-    def _process_kinds_on_init(self, arg_kinds):
+    def _process_kinds_on_init(self, arg_kinds: Iterable[int]) -> None:
         self.is_var_arg = False
         self.is_kw_arg = False
         seen_named = False