Memoize some rules to avoid exponential time in deep nesting

Additionally, fix the script to calculate the nesting level using the
generated extension.

Author: pablogsal

Grammar/python.gram

@@ -461,7 +461,7 @@ star_targets[expr_ty]:
     | a=star_target b=(',' c=star_target { c })* [','] {
         _Py_Tuple(CHECK(seq_insert_in_front(p, a, b)), Store, EXTRA) }
 star_targets_seq[asdl_seq*]: a=','.star_target+ [','] { a }
-star_target[expr_ty]:
+star_target[expr_ty] (memo):
     | '*' a=bitwise_or { _Py_Starred(CHECK(set_expr_context(p, a, Store)), Store, EXTRA) }
     | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) }
     | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) }
@@ -482,7 +482,7 @@ ann_assign_subscript_attribute_target[expr_ty]:
     | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) }
 
 del_targets[asdl_seq*]: a=','.del_target+ [','] { a }
-del_target[expr_ty]:
+del_target[expr_ty] (memo):
     | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Del, EXTRA) }
     | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Del, EXTRA) }
     | del_t_atom
@@ -493,7 +493,7 @@ del_t_atom[expr_ty]:
     | '[' a=[del_targets] ']' { _Py_List(a, Del, EXTRA) }
 
 targets[asdl_seq*]: a=','.target+ [','] { a }
-target[expr_ty]:
+target[expr_ty] (memo):
     | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) }
     | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) }
     | t_atom

Parser/pegen/parse.c

@@ -7766,6 +7766,8 @@ static expr_ty
 star_target_rule(Parser *p)
 {
     expr_ty res = NULL;
+    if (is_memoized(p, star_target_type, &res))
+        return res;
     int mark = p->mark;
     if (p->mark == p->fill && fill_token(p) < 0) {
         return NULL;
@@ -7875,6 +7877,7 @@ star_target_rule(Parser *p)
     }
     res = NULL;
   done:
+    insert_memo(p, mark, star_target_type, res);
     return res;
 }
 
@@ -8168,6 +8171,8 @@ static expr_ty
 del_target_rule(Parser *p)
 {
     expr_ty res = NULL;
+    if (is_memoized(p, del_target_type, &res))
+        return res;
     int mark = p->mark;
     if (p->mark == p->fill && fill_token(p) < 0) {
         return NULL;
@@ -8252,6 +8257,7 @@ del_target_rule(Parser *p)
     }
     res = NULL;
   done:
+    insert_memo(p, mark, del_target_type, res);
     return res;
 }
 
@@ -8399,6 +8405,8 @@ static expr_ty
 target_rule(Parser *p)
 {
     expr_ty res = NULL;
+    if (is_memoized(p, target_type, &res))
+        return res;
     int mark = p->mark;
     if (p->mark == p->fill && fill_token(p) < 0) {
         return NULL;
@@ -8483,6 +8491,7 @@ target_rule(Parser *p)
     }
     res = NULL;
   done:
+    insert_memo(p, mark, target_type, res);
     return res;
 }
 

Tools/peg_generator/scripts/find_max_nesting.py

@@ -19,9 +19,7 @@
 from pathlib import Path
 from typing import Any
 
-sys.path.insert(0, ".")
-from pegen.build import build_parser
-from pegen.testutil import generate_parser, generate_parser_c_extension, make_parser, parse_string
+from peg_parser import parse_string
 
 GRAMMAR_FILE = "data/python.gram"
 INITIAL_NESTING_DEPTH = 10
@@ -32,47 +30,31 @@
 ENDC = "\033[0m"
 
 
-def check_nested_expr(nesting_depth: int, parser: Any, language: str) -> bool:
+def check_nested_expr(nesting_depth: int) -> bool:
     expr = f"{'(' * nesting_depth}0{')' * nesting_depth}"
 
     try:
-        if language == "Python":
-            parse_string(expr, parser)
-        else:
-            parser.parse_string(expr)
-
-        print(f"({language}) Nesting depth of {nesting_depth} is successful")
-
+        parse_string(expr)
+        print(f"Nesting depth of {nesting_depth} is successful")
         return True
     except Exception as err:
-        print(f"{FAIL}({language}) Failed with nesting depth of {nesting_depth}{ENDC}")
+        print(f"{FAIL}(Failed with nesting depth of {nesting_depth}{ENDC}")
         print(f"{FAIL}\t{err}{ENDC}")
         return False
 
 
 def main() -> None:
     print(f"Testing {GRAMMAR_FILE} starting at nesting depth of {INITIAL_NESTING_DEPTH}...")
 
-    with TemporaryDirectory() as tmp_dir:
-        nesting_depth = INITIAL_NESTING_DEPTH
-        rules, parser, tokenizer = build_parser(GRAMMAR_FILE)
-        python_parser = generate_parser(rules)
-        c_parser = generate_parser_c_extension(rules, Path(tmp_dir))
-
-        c_succeeded = True
-        python_succeeded = True
-
-        while c_succeeded or python_succeeded:
-            expr = f"{'(' * nesting_depth}0{')' * nesting_depth}"
-
-            if c_succeeded:
-                c_succeeded = check_nested_expr(nesting_depth, c_parser, "C")
-            if python_succeeded:
-                python_succeeded = check_nested_expr(nesting_depth, python_parser, "Python")
-
-            nesting_depth += NESTED_INCR_AMT
+    nesting_depth = INITIAL_NESTING_DEPTH
+    succeeded = True
+    while succeeded:
+        expr = f"{'(' * nesting_depth}0{')' * nesting_depth}"
+        if succeeded:
+            succeeded = check_nested_expr(nesting_depth)
+        nesting_depth += NESTED_INCR_AMT
 
-        sys.exit(1)
+    sys.exit(1)
 
 
 if __name__ == "__main__":