Support '# type: ignore <tag>'

Grammar/python.gram

@@ -28,7 +28,7 @@ _PyPegen_parse(Parser *p)
 
 // The end
 '''
-file[mod_ty]: a=[statements] ENDMARKER { Module(a, NULL, p->arena) }
+file[mod_ty]: a=[statements] ENDMARKER { _PyPegen_make_module(p, a) }
 interactive[mod_ty]: a=statement_newline { Interactive(a, p->arena) }
 eval[mod_ty]: a=expressions NEWLINE* ENDMARKER { Expression(a, p->arena) }
 func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression { FunctionType(a, b, p->arena) }

Lib/test/test_type_comments.py

@@ -313,7 +313,6 @@ def test_vardecl(self):
         tree = self.classic_parse(vardecl)
         self.assertEqual(tree.body[0].type_comment, None)
 
-    @support.skip_if_new_parser("Pegen does not support `# type: ignore` yet")
     def test_ignores(self):
         for tree in self.parse_all(ignores):
             self.assertEqual(

Parser/pegen/parse.c

@@ -677,7 +677,7 @@ file_rule(Parser *p)
             (endmarker_var = _PyPegen_endmarker_token(p))
         )
         {
-            res = Module ( a , NULL , p -> arena );
+            res = _PyPegen_make_module ( p , a );
             if (res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 return NULL;

Parser/pegen/pegen.c

@@ -553,14 +553,63 @@ _get_keyword_or_name_type(Parser *p, const char *name, int name_len)
     return NAME;
 }
 
+static int
+growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
+    assert(initial_size > 0);
+    arr->items = PyMem_Malloc(initial_size * sizeof(*arr->items));
+    arr->size = initial_size;
+    arr->num_items = 0;
+
+    return arr->items != NULL;
+}
+
+static int
+growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
+    if (arr->num_items >= arr->size) {
+        size_t new_size = arr->size * 2;
+        void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items));
+        if (!new_items_array) {
+            return 0;
+        }
+        arr->items = new_items_array;
+        arr->size = new_size;
+    }
+
+    arr->items[arr->num_items].lineno = lineno;
+    arr->items[arr->num_items].comment = comment;  // Take ownership
+    arr->num_items++;
+    return 1;
+}
+
+static void
+growable_comment_array_deallocate(growable_comment_array *arr) {
+    for (unsigned i = 0; i < arr->num_items; i++) {
+        PyMem_Free(arr->items[i].comment);
+    }
+    PyMem_Free(arr->items);
+}
+
 int
 _PyPegen_fill_token(Parser *p)
 {
     const char *start, *end;
     int type = PyTokenizer_Get(p->tok, &start, &end);
 
-    // Skip '# type: ignore'; TODO: collect these like parsetok.c
+    // Record and skip '# type: ignore' comments
     while (type == TYPE_IGNORE) {
+        Py_ssize_t len = end - start;
+        char *tag = PyMem_Malloc(len + 1);
+        if (tag == NULL) {
+            PyErr_NoMemory();
+            return -1;
+        }
+        strncpy(tag, start, len);
+        tag[len] = '\0';
+        // Ownership of tag passes to the growable array
+        if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
+            PyErr_NoMemory();
+            return -1;
+        }
         type = PyTokenizer_Get(p->tok, &start, &end);
     }
 
@@ -945,6 +994,7 @@ _PyPegen_Parser_Free(Parser *p)
         PyMem_Free(p->tokens[i]);
     }
     PyMem_Free(p->tokens);
+    growable_comment_array_deallocate(&p->type_ignore_comments);
     PyMem_Free(p);
 }
 
@@ -987,13 +1037,19 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
         PyMem_Free(p);
         return (Parser *) PyErr_NoMemory();
     }
-    p->tokens[0] = PyMem_Malloc(sizeof(Token));
+    p->tokens[0] = PyMem_Calloc(1, sizeof(Token));
     if (!p->tokens) {
         PyMem_Free(p->tokens);
         PyMem_Free(p);
         return (Parser *) PyErr_NoMemory();
     }
-    memset(p->tokens[0], '\0', sizeof(Token));
+    if (!growable_comment_array_init(&p->type_ignore_comments, 10)) {
+        PyMem_Free(p->tokens[0]);
+        PyMem_Free(p->tokens);
+        PyMem_Free(p);
+        return (Parser *) PyErr_NoMemory();
+    }
+
     p->mark = 0;
     p->fill = 0;
     p->size = 1;
@@ -1993,3 +2049,28 @@ _PyPegen_concatenate_strings(Parser *p, asdl_seq *strings)
     }
     return NULL;
 }
+
+mod_ty
+_PyPegen_make_module(Parser *p, asdl_seq *a) {
+    asdl_seq *type_ignores = NULL;
+    Py_ssize_t num = p->type_ignore_comments.num_items;
+    if (num > 0) {
+        // Turn the raw (comment, lineno) pairs into TypeIgnore objects in the arena
+        type_ignores = _Py_asdl_seq_new(num, p->arena);
+        if (type_ignores == NULL) {
+            return NULL;
+        }
+        for (int i = 0; i < num; i++) {
+            PyObject *tag = _PyPegen_new_type_comment(p, p->type_ignore_comments.items[i].comment);
+            if (tag == NULL) {
+                return NULL;
+            }
+            type_ignore_ty ti = TypeIgnore(p->type_ignore_comments.items[i].lineno, tag, p->arena);
+            if (ti == NULL) {
+                return NULL;
+            }
+            asdl_seq_SET(type_ignores, i, ti);
+        }
+    }
+    return Module(a, type_ignores, p->arena);
+}

Parser/pegen/pegen.h

@@ -43,6 +43,16 @@ typedef struct {
     int type;
 } KeywordToken;
 
+
+typedef struct {
+    struct {
+        int lineno;
+        char *comment;  // The " <tag>" in "# type: ignore <tag>"
+    } *items;
+    size_t size;
+    size_t num_items;
+} growable_comment_array;
+
 typedef struct {
     struct tok_state *tok;
     Token **tokens;
@@ -59,6 +69,7 @@ typedef struct {
     int starting_col_offset;
     int error_indicator;
     int flags;
+    growable_comment_array type_ignore_comments;
 } Parser;
 
 typedef struct {
@@ -198,6 +209,7 @@ expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_seq *);
 asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *);
 void *_PyPegen_arguments_parsing_error(Parser *, expr_ty);
 int _PyPegen_check_barry_as_flufl(Parser *);
+mod_ty _PyPegen_make_module(Parser *, asdl_seq *);
 
 void *_PyPegen_parse(Parser *);