Skip to content

Commit

Permalink
Make most fastparser errors non-fatal (#2689)
Browse files Browse the repository at this point in the history 
Instead of throwing an exception up to the parse function when encountering an error, the fast parser will now report it to the errors object and keep parsing. These errors are still considered blocking errors, so type checking will not continue past the parsing stage. This means that mypy will show you all your parse errors, instead of showing them to you one at a time as you fix them.

Fixes #2685.
  • Loading branch information
@ddfisher @gvanrossum
ddfisher authored and gvanrossum committed Jan 17, 2017
1 parent 2de2f4c commit 9141362
Show file tree
Hide file tree
Showing 4 changed files with 115 additions and 102 deletions.
125 changes: 65 additions & 60 deletions mypy/fastparse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,36 +56,40 @@ def parse(source: Union[str, bytes], fnam: str = None, errors: Errors = None,
The pyversion (major, minor) argument determines the Python syntax variant.
"""
if errors is None:
errors = Errors()
errors.set_file('<input>' if fnam is None else fnam)
is_stub_file = bool(fnam) and fnam.endswith('.pyi')
try:
assert pyversion[0] >= 3 or is_stub_file
ast = ast35.parse(source, fnam, 'exec')

tree = ASTConverter(pyversion=pyversion,
is_stub=is_stub_file,
errors=errors,
custom_typing_module=custom_typing_module,
).visit(ast)
tree.path = fnam
tree.is_stub = is_stub_file
return tree
except (SyntaxError, TypeCommentParseError) as e:
except SyntaxError as e:
if errors:
errors.set_file('<input>' if fnam is None else fnam)
errors.report(e.lineno, e.offset, e.msg)
else:
raise

return MypyFile([], [], False, set())


def parse_type_comment(type_comment: str, line: int) -> Type:
def parse_type_comment(type_comment: str, line: int, errors: Errors) -> Optional[Type]:
try:
typ = ast35.parse(type_comment, '<type_comment>', 'eval')
except SyntaxError as e:
raise TypeCommentParseError(TYPE_COMMENT_SYNTAX_ERROR, line, e.offset)
errors.report(line, e.offset, TYPE_COMMENT_SYNTAX_ERROR)
return None
else:
assert isinstance(typ, ast35.Expression)
return TypeConverter(line=line).visit(typ.body)
return TypeConverter(errors, line=line).visit(typ.body)


def with_line(f: Callable[['ASTConverter', T], U]) -> Callable[['ASTConverter', T], U]:
Expand All @@ -108,14 +112,19 @@ class ASTConverter(ast35.NodeTransformer):
def __init__(self,
pyversion: Tuple[int, int],
is_stub: bool,
errors: Errors,
custom_typing_module: str = None) -> None:
self.class_nesting = 0
self.imports = [] # type: List[ImportBase]

self.pyversion = pyversion
self.is_stub = is_stub
self.errors = errors
self.custom_typing_module = custom_typing_module

def fail(self, msg: str, line: int, column: int) -> None:
self.errors.report(line, column, msg)

def generic_visit(self, node: ast35.AST) -> None:
raise RuntimeError('AST node not implemented: ' + str(type(node)))

Expand Down Expand Up @@ -270,27 +279,30 @@ def do_func_def(self, n: Union[ast35.FunctionDef, ast35.AsyncFunctionDef],
if n.type_comment is not None:
try:
func_type_ast = ast35.parse(n.type_comment, '<func_type>', 'func_type')
assert isinstance(func_type_ast, ast35.FunctionType)
# for ellipsis arg
if (len(func_type_ast.argtypes) == 1 and
isinstance(func_type_ast.argtypes[0], ast35.Ellipsis)):
arg_types = [a.type_annotation if a.type_annotation is not None else AnyType()
for a in args]
else:
translated_args = (TypeConverter(self.errors, line=n.lineno)
.translate_expr_list(func_type_ast.argtypes))
arg_types = [a if a is not None else AnyType()
for a in translated_args]
return_type = TypeConverter(self.errors,
line=n.lineno).visit(func_type_ast.returns)

# add implicit self type
if self.in_class() and len(arg_types) < len(args):
arg_types.insert(0, AnyType())
except SyntaxError:
raise TypeCommentParseError(TYPE_COMMENT_SYNTAX_ERROR, n.lineno, n.col_offset)
assert isinstance(func_type_ast, ast35.FunctionType)
# for ellipsis arg
if (len(func_type_ast.argtypes) == 1 and
isinstance(func_type_ast.argtypes[0], ast35.Ellipsis)):
arg_types = [a.type_annotation if a.type_annotation is not None else AnyType()
for a in args]
else:
translated_args = (TypeConverter(line=n.lineno)
.translate_expr_list(func_type_ast.argtypes))
arg_types = [a if a is not None else AnyType()
for a in translated_args]
return_type = TypeConverter(line=n.lineno).visit(func_type_ast.returns)

# add implicit self type
if self.in_class() and len(arg_types) < len(args):
arg_types.insert(0, AnyType())
self.fail(TYPE_COMMENT_SYNTAX_ERROR, n.lineno, n.col_offset)
arg_types = [AnyType() for _ in args]
return_type = AnyType()
else:
arg_types = [a.type_annotation for a in args]
return_type = TypeConverter(line=n.lineno).visit(n.returns)
return_type = TypeConverter(self.errors, line=n.lineno).visit(n.returns)

for arg, arg_type in zip(args, arg_types):
self.set_type_optional(arg_type, arg.initializer)
Expand All @@ -301,16 +313,17 @@ def do_func_def(self, n: Union[ast35.FunctionDef, ast35.AsyncFunctionDef],
func_type = None
if any(arg_types) or return_type:
if len(arg_types) > len(arg_kinds):
raise FastParserError('Type signature has too many arguments', n.lineno, offset=0)
if len(arg_types) < len(arg_kinds):
raise FastParserError('Type signature has too few arguments', n.lineno, offset=0)
func_type = CallableType([a if a is not None else
AnyType(implicit=True) for a in arg_types],
arg_kinds,
arg_names,
return_type if return_type is not None else
AnyType(implicit=True),
None)
self.fail('Type signature has too many arguments', n.lineno, 0)
elif len(arg_types) < len(arg_kinds):
self.fail('Type signature has too few arguments', n.lineno, 0)
else:
func_type = CallableType([a if a is not None else
AnyType(implicit=True) for a in arg_types],
arg_kinds,
arg_names,
return_type if return_type is not None else
AnyType(implicit=True),
None)

func_def = FuncDef(n.name,
args,
Expand Down Expand Up @@ -345,7 +358,7 @@ def set_type_optional(self, type: Type, initializer: Expression) -> None:

def transform_args(self, args: ast35.arguments, line: int) -> List[Argument]:
def make_argument(arg: ast35.arg, default: Optional[ast35.expr], kind: int) -> Argument:
arg_type = TypeConverter(line=line).visit(arg.annotation)
arg_type = TypeConverter(self.errors, line=line).visit(arg.annotation)
return Argument(Var(arg.arg), arg_type, self.visit(default), kind)

new_args = []
Expand Down Expand Up @@ -432,14 +445,13 @@ def visit_Assign(self, n: ast35.Assign) -> AssignmentStmt:
else:
new_syntax = False
if new_syntax and self.pyversion < (3, 6):
raise TypeCommentParseError('Variable annotation syntax is only '
'suppoted in Python 3.6, use type '
'comment instead', n.lineno, n.col_offset)
self.fail('Variable annotation syntax is only supported in Python 3.6, '
'use type comment instead', n.lineno, n.col_offset)
# typed_ast prevents having both type_comment and annotation.
if n.type_comment is not None:
typ = parse_type_comment(n.type_comment, n.lineno)
typ = parse_type_comment(n.type_comment, n.lineno, self.errors)
elif new_syntax:
typ = TypeConverter(line=n.lineno).visit(n.annotation) # type: ignore
typ = TypeConverter(self.errors, line=n.lineno).visit(n.annotation) # type: ignore
typ.column = n.annotation.col_offset
if n.value is None: # always allow 'x: int'
rvalue = TempNode(AnyType()) # type: Expression
Expand Down Expand Up @@ -746,8 +758,8 @@ def is_star2arg(k: ast35.keyword) -> bool:
@with_line
def visit_Num(self, n: ast35.Num) -> Union[IntExpr, FloatExpr, ComplexExpr]:
if getattr(n, 'contains_underscores', None) and self.pyversion < (3, 6):
raise FastParserError('Underscores in numeric literals are only '
'supported in Python 3.6', n.lineno, n.col_offset)
self.fail('Underscores in numeric literals are only supported in Python 3.6',
n.lineno, n.col_offset)
if isinstance(n.n, int):
return IntExpr(n.n)
elif isinstance(n.n, float):
Expand Down Expand Up @@ -845,18 +857,22 @@ def visit_Index(self, n: ast35.Index) -> Node:


class TypeConverter(ast35.NodeTransformer):
def __init__(self, line: int = -1) -> None:
def __init__(self, errors: Errors, line: int = -1) -> None:
self.errors = errors
self.line = line

def fail(self, msg: str, line: int, column: int) -> None:
self.errors.report(line, column, msg)

def visit_raw_str(self, s: str) -> Type:
# An escape hatch that allows the AST walker in fastparse2 to
# directly hook into the Python 3.5 type converter in some cases
# without needing to create an intermediary `ast35.Str` object.
return parse_type_comment(s.strip(), line=self.line)
return parse_type_comment(s.strip(), self.line, self.errors) or AnyType()

def generic_visit(self, node: ast35.AST) -> None:
raise TypeCommentParseError(TYPE_COMMENT_AST_ERROR, self.line,
getattr(node, 'col_offset', -1))
def generic_visit(self, node: ast35.AST) -> Type: # type: ignore
self.fail(TYPE_COMMENT_AST_ERROR, self.line, getattr(node, 'col_offset', -1))
return AnyType()

def visit_NoneType(self, n: Any) -> Type:
return None
Expand All @@ -872,13 +888,13 @@ def visit_NameConstant(self, n: ast35.NameConstant) -> Type:

# Str(string s)
def visit_Str(self, n: ast35.Str) -> Type:
return parse_type_comment(n.s.strip(), line=self.line)
return parse_type_comment(n.s.strip(), self.line, self.errors) or AnyType()

# Subscript(expr value, slice slice, expr_context ctx)
def visit_Subscript(self, n: ast35.Subscript) -> Type:
if not isinstance(n.slice, ast35.Index):
raise TypeCommentParseError(TYPE_COMMENT_SYNTAX_ERROR, self.line,
getattr(n, 'col_offset', -1))
self.fail(TYPE_COMMENT_SYNTAX_ERROR, self.line, getattr(n, 'col_offset', -1))
return AnyType()

value = self.visit(n.value)

Expand Down Expand Up @@ -914,14 +930,3 @@ def visit_Ellipsis(self, n: ast35.Ellipsis) -> Type:
# List(expr* elts, expr_context ctx)
def visit_List(self, n: ast35.List) -> Type:
return TypeList(self.translate_expr_list(n.elts), line=self.line)


class TypeCommentParseError(Exception):
def __init__(self, msg: str, lineno: int, offset: int) -> None:
self.msg = msg
self.lineno = lineno
self.offset = offset


class FastParserError(TypeCommentParseError):
pass
Loading

0 comments on commit 9141362

Please sign in to comment.