Skip to content

Commit c001c09

Browse files
authored
bpo-40334: Support type comments (GH-19780)
This implements full support for # type: <type> comments, # type: ignore <stuff> comments, and the func_type parsing mode for ast.parse() and compile(). Closes we-like-parsers#95. (For now, you need to use the master branch of mypy, since another issue unique to 3.9 had to be fixed there, and there's no mypy release yet.) The only thing missing is `feature_version=N`, which is being tracked in we-like-parsers#124.
1 parent efb8dd5 commit c001c09

File tree

6 files changed

+2977
-1858
lines changed

6 files changed

+2977
-1858
lines changed

Grammar/python.gram

Lines changed: 77 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ _PyPegen_parse(Parser *p)
1717
result = interactive_rule(p);
1818
} else if (p->start_rule == Py_eval_input) {
1919
result = eval_rule(p);
20+
} else if (p->start_rule == Py_func_type_input) {
21+
result = func_type_rule(p);
2022
} else if (p->start_rule == Py_fstring_input) {
2123
result = fstring_rule(p);
2224
}
@@ -26,11 +28,20 @@ _PyPegen_parse(Parser *p)
2628

2729
// The end
2830
'''
29-
file[mod_ty]: a=[statements] ENDMARKER { Module(a, NULL, p->arena) }
31+
file[mod_ty]: a=[statements] ENDMARKER { _PyPegen_make_module(p, a) }
3032
interactive[mod_ty]: a=statement_newline { Interactive(a, p->arena) }
3133
eval[mod_ty]: a=expressions NEWLINE* ENDMARKER { Expression(a, p->arena) }
34+
func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMARKER { FunctionType(a, b, p->arena) }
3235
fstring[expr_ty]: star_expressions
3336

37+
# type_expressions allow */** but ignore them
38+
type_expressions[asdl_seq*]:
39+
| a=','.expression+ ',' '*' b=expression ',' '**' c=expression {
40+
_PyPegen_seq_append_to_end(p, CHECK(_PyPegen_seq_append_to_end(p, a, b)), c) }
41+
| a=','.expression+ ',' '*' b=expression { _PyPegen_seq_append_to_end(p, a, b) }
42+
| a=','.expression+ ',' '**' b=expression { _PyPegen_seq_append_to_end(p, a, b) }
43+
| ','.expression+
44+
3445
statements[asdl_seq*]: a=statement+ { _PyPegen_seq_flatten(p, a) }
3546
statement[asdl_seq*]: a=compound_stmt { _PyPegen_singleton_seq(p, a) } | simple_stmt
3647
statement_newline[asdl_seq*]:
@@ -73,8 +84,8 @@ assignment:
7384
| a=('(' b=inside_paren_ann_assign_target ')' { b }
7485
| ann_assign_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] {
7586
_Py_AnnAssign(a, b, c, 0, EXTRA)}
76-
| a=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) {
77-
_Py_Assign(a, b, NULL, EXTRA) }
87+
| a=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) tc=[TYPE_COMMENT] {
88+
_Py_Assign(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
7889
| a=target b=augassign c=(yield_expr | star_expressions) {
7990
_Py_AugAssign(a, b->kind, c, EXTRA) }
8091
| invalid_assignment
@@ -145,14 +156,14 @@ while_stmt[stmt_ty]:
145156
| 'while' a=named_expression ':' b=block c=[else_block] { _Py_While(a, b, c, EXTRA) }
146157

147158
for_stmt[stmt_ty]:
148-
| is_async=[ASYNC] 'for' t=star_targets 'in' ex=star_expressions ':' b=block el=[else_block] {
149-
(is_async ? _Py_AsyncFor : _Py_For)(t, ex, b, el, NULL, EXTRA) }
159+
| is_async=[ASYNC] 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
160+
(is_async ? _Py_AsyncFor : _Py_For)(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) }
150161

151162
with_stmt[stmt_ty]:
152163
| is_async=[ASYNC] 'with' '(' a=','.with_item+ ')' ':' b=block {
153164
(is_async ? _Py_AsyncWith : _Py_With)(a, b, NULL, EXTRA) }
154-
| is_async=[ASYNC] 'with' a=','.with_item+ ':' b=block {
155-
(is_async ? _Py_AsyncWith : _Py_With)(a, b, NULL, EXTRA) }
165+
| is_async=[ASYNC] 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
166+
(is_async ? _Py_AsyncWith : _Py_With)(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
156167
with_item[withitem_ty]:
157168
| e=expression o=['as' t=target { t }] { _Py_withitem(e, o, p->arena) }
158169

@@ -177,43 +188,74 @@ function_def[stmt_ty]:
177188
| function_def_raw
178189

179190
function_def_raw[stmt_ty]:
180-
| is_async=[ASYNC] 'def' n=NAME '(' params=[params] ')' a=['->' z=annotation { z }] ':' b=block {
191+
| is_async=[ASYNC] 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block {
181192
(is_async ? _Py_AsyncFunctionDef : _Py_FunctionDef)(n->v.Name.id,
182193
(params) ? params : CHECK(_PyPegen_empty_arguments(p)),
183-
b, NULL, a, NULL, EXTRA) }
194+
b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) }
195+
func_type_comment[PyObject*]:
196+
| NEWLINE t=TYPE_COMMENT &(NEWLINE INDENT) { t } # Must be followed by indented block
197+
| invalid_double_type_comments
198+
| TYPE_COMMENT
184199

185200
params[arguments_ty]:
186201
| invalid_parameters
187202
| parameters
203+
188204
parameters[arguments_ty]:
189-
| a=slash_without_default b=[',' x=plain_names { x }] c=[',' y=names_with_default { y }] d=[',' z=[star_etc] { z }] {
205+
| a=slash_no_default b=param_no_default* c=param_with_default* d=[star_etc] {
190206
_PyPegen_make_arguments(p, a, NULL, b, c, d) }
191-
| a=slash_with_default b=[',' y=names_with_default { y }] c=[',' z=[star_etc] { z }] {
207+
| a=slash_with_default b=param_with_default* c=[star_etc] {
192208
_PyPegen_make_arguments(p, NULL, a, NULL, b, c) }
193-
| a=plain_names b=[',' y=names_with_default { y }] c=[',' z=[star_etc] { z }] {
209+
| a=param_no_default+ b=param_with_default* c=[star_etc] {
194210
_PyPegen_make_arguments(p, NULL, NULL, a, b, c) }
195-
| a=names_with_default b=[',' z=[star_etc] { z }] { _PyPegen_make_arguments(p, NULL, NULL, NULL, a, b)}
211+
| a=param_with_default+ b=[star_etc] { _PyPegen_make_arguments(p, NULL, NULL, NULL, a, b)}
196212
| a=star_etc { _PyPegen_make_arguments(p, NULL, NULL, NULL, NULL, a) }
197-
slash_without_default[asdl_seq*]: a=plain_names ',' '/' { a }
198-
slash_with_default[SlashWithDefault*]: a=[n=plain_names ',' { n }] b=names_with_default ',' '/' {
199-
_PyPegen_slash_with_default(p, a, b) }
213+
214+
# Some duplication here because we can't write (',' | &')'),
215+
# which is because we don't support empty alternatives (yet).
216+
#
217+
slash_no_default[asdl_seq*]:
218+
| a=param_no_default+ '/' ',' { a }
219+
| a=param_no_default+ '/' &')' { a }
220+
slash_with_default[SlashWithDefault*]:
221+
| a=param_no_default* b=param_with_default+ '/' ',' { _PyPegen_slash_with_default(p, a, b) }
222+
| a=param_no_default* b=param_with_default+ '/' &')' { _PyPegen_slash_with_default(p, a, b) }
223+
200224
star_etc[StarEtc*]:
201-
| '*' a=plain_name b=name_with_optional_default* c=[',' d=kwds { d }] [','] {
225+
| '*' a=param_no_default b=param_maybe_default* c=[kwds] {
202226
_PyPegen_star_etc(p, a, b, c) }
203-
| '*' b=name_with_optional_default+ c=[',' d=kwds { d }] [','] {
227+
| '*' ',' b=param_maybe_default+ c=[kwds] {
204228
_PyPegen_star_etc(p, NULL, b, c) }
205-
| a=kwds [','] { _PyPegen_star_etc(p, NULL, NULL, a) }
206-
name_with_optional_default[NameDefaultPair*]:
207-
| ',' a=plain_name b=['=' e=expression { e }] { _PyPegen_name_default_pair(p, a, b) }
208-
names_with_default[asdl_seq*]: a=','.name_with_default+ { a }
209-
name_with_default[NameDefaultPair*]:
210-
| n=plain_name '=' e=expression { _PyPegen_name_default_pair(p, n, e) }
211-
plain_names[asdl_seq*] (memo): a=','.(plain_name !'=')+ { a }
212-
plain_name[arg_ty]:
213-
| a=NAME b=[':' z=annotation { z }] { _Py_arg(a->v.Name.id, b, NULL, EXTRA) }
229+
| a=kwds { _PyPegen_star_etc(p, NULL, NULL, a) }
230+
214231
kwds[arg_ty]:
215-
| '**' a=plain_name { a }
216-
annotation[expr_ty]: expression
232+
| '**' a=param_no_default { a }
233+
234+
# One parameter. This *includes* a following comma and type comment.
235+
#
236+
# There are three styles:
237+
# - No default
238+
# - With default
239+
# - Maybe with default
240+
#
241+
# There are two alternative forms of each, to deal with type comments:
242+
# - Ends in a comma followed by an optional type comment
243+
# - No comma, optional type comment, must be followed by close paren
244+
# The latter form is for a final parameter without trailing comma.
245+
#
246+
param_no_default[arg_ty]:
247+
| a=param ',' tc=TYPE_COMMENT? { _PyPegen_add_type_comment_to_arg(p, a, tc) }
248+
| a=param tc=TYPE_COMMENT? &')' { _PyPegen_add_type_comment_to_arg(p, a, tc) }
249+
param_with_default[NameDefaultPair*]:
250+
| a=param c=default ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) }
251+
| a=param c=default tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) }
252+
param_maybe_default[NameDefaultPair*]:
253+
| a=param c=default? ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) }
254+
| a=param c=default? tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) }
255+
param[arg_ty]: a=NAME b=annotation? { _Py_arg(a->v.Name.id, b, NULL, EXTRA) }
256+
257+
annotation[expr_ty]: ':' a=expression { a }
258+
default[expr_ty]: '=' a=expression { a }
217259

218260
decorators[asdl_seq*]: a=('@' f=named_expression NEWLINE { f })+ { a }
219261

@@ -284,10 +326,10 @@ lambda_star_etc[StarEtc*]:
284326
_PyPegen_star_etc(p, NULL, b, c) }
285327
| a=lambda_kwds [','] { _PyPegen_star_etc(p, NULL, NULL, a) }
286328
lambda_name_with_optional_default[NameDefaultPair*]:
287-
| ',' a=lambda_plain_name b=['=' e=expression { e }] { _PyPegen_name_default_pair(p, a, b) }
329+
| ',' a=lambda_plain_name b=['=' e=expression { e }] { _PyPegen_name_default_pair(p, a, b, NULL) }
288330
lambda_names_with_default[asdl_seq*]: a=','.lambda_name_with_default+ { a }
289331
lambda_name_with_default[NameDefaultPair*]:
290-
| n=lambda_plain_name '=' e=expression { _PyPegen_name_default_pair(p, n, e) }
332+
| n=lambda_plain_name '=' e=expression { _PyPegen_name_default_pair(p, n, e, NULL) }
291333
lambda_plain_names[asdl_seq*]: a=','.(lambda_plain_name !'=')+ { a }
292334
lambda_plain_name[arg_ty]: a=NAME { _Py_arg(a->v.Name.id, NULL, NULL, EXTRA) }
293335
lambda_kwds[arg_ty]: '**' a=lambda_plain_name { a }
@@ -552,5 +594,8 @@ invalid_comprehension:
552594
| ('[' | '(' | '{') '*' expression for_if_clauses {
553595
RAISE_SYNTAX_ERROR("iterable unpacking cannot be used in comprehension") }
554596
invalid_parameters:
555-
| [plain_names ','] (slash_with_default | names_with_default) ',' plain_names {
597+
| param_no_default* (slash_with_default | param_with_default+) param_no_default {
556598
RAISE_SYNTAX_ERROR("non-default argument follows default argument") }
599+
invalid_double_type_comments:
600+
| TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT {
601+
RAISE_SYNTAX_ERROR("Cannot have two type comments on def") }

Lib/test/test_type_comments.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,6 @@ def favk(
219219
"""
220220

221221

222-
@support.skip_if_new_parser("Pegen does not support type comments yet")
223222
class TypeCommentTests(unittest.TestCase):
224223

225224
lowest = 4 # Lowest minor version supported
@@ -253,6 +252,7 @@ def test_funcdef(self):
253252
self.assertEqual(tree.body[0].type_comment, None)
254253
self.assertEqual(tree.body[1].type_comment, None)
255254

255+
@support.skip_if_new_parser("Pegen does not support feature_version yet")
256256
def test_asyncdef(self):
257257
for tree in self.parse_all(asyncdef, minver=5):
258258
self.assertEqual(tree.body[0].type_comment, "() -> int")
@@ -261,22 +261,27 @@ def test_asyncdef(self):
261261
self.assertEqual(tree.body[0].type_comment, None)
262262
self.assertEqual(tree.body[1].type_comment, None)
263263

264+
@support.skip_if_new_parser("Pegen does not support feature_version yet")
264265
def test_asyncvar(self):
265266
for tree in self.parse_all(asyncvar, maxver=6):
266267
pass
267268

269+
@support.skip_if_new_parser("Pegen does not support feature_version yet")
268270
def test_asynccomp(self):
269271
for tree in self.parse_all(asynccomp, minver=6):
270272
pass
271273

274+
@support.skip_if_new_parser("Pegen does not support feature_version yet")
272275
def test_matmul(self):
273276
for tree in self.parse_all(matmul, minver=5):
274277
pass
275278

279+
@support.skip_if_new_parser("Pegen does not support feature_version yet")
276280
def test_fstring(self):
277281
for tree in self.parse_all(fstring, minver=6):
278282
pass
279283

284+
@support.skip_if_new_parser("Pegen does not support feature_version yet")
280285
def test_underscorednumber(self):
281286
for tree in self.parse_all(underscorednumber, minver=6):
282287
pass

0 commit comments

Comments
 (0)