Update pegen to use the latest upstream developments

Lib/test/test_peg_generator/test_c_parser.py

@@ -11,8 +11,8 @@
 from test.support import os_helper
 from test.support.script_helper import assert_python_ok
 
-_py_cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST')
-_pgo_flag = sysconfig.get_config_var('PGO_PROF_USE_FLAG')
+_py_cflags_nodist = sysconfig.get_config_var("PY_CFLAGS_NODIST")
+_pgo_flag = sysconfig.get_config_var("PGO_PROF_USE_FLAG")
 if _pgo_flag and _py_cflags_nodist and _pgo_flag in _py_cflags_nodist:
     raise unittest.SkipTest("peg_generator test disabled under PGO build")
 
@@ -458,3 +458,28 @@ def test_soft_keywords_lookahead(self) -> None:
         self.check_input_strings_for_grammar(valid_cases, invalid_cases)
         """
         self.run_test(grammar_source, test_source)
+
+    def test_forced(self) -> None:
+        grammar_source = """
+        start: NAME &&':' | NAME
+        """
+        test_source = """
+        self.assertEqual(parse.parse_string("number :", mode=0), None)
+        with self.assertRaises(SyntaxError) as e:
+            parse.parse_string("a", mode=0)
+        self.assertIn("expected ':'", str(e.exception))
+        """
+        self.run_test(grammar_source, test_source)
+
+    def test_forced_with_group(self) -> None:
+        grammar_source = """
+        start: NAME &&(':' | ';') | NAME
+        """
+        test_source = """
+        self.assertEqual(parse.parse_string("number :", mode=0), None)
+        self.assertEqual(parse.parse_string("number ;", mode=0), None)
+        with self.assertRaises(SyntaxError) as e:
+            parse.parse_string("a", mode=0)
+        self.assertIn("expected (':' | ';')", e.exception.args[0])
+        """
+        self.run_test(grammar_source, test_source)

Lib/test/test_peg_generator/test_first_sets.py

@@ -3,8 +3,8 @@
 from test import test_tools
 from typing import Dict, Set
 
-test_tools.skip_if_missing('peg_generator')
-with test_tools.imports_under_tool('peg_generator'):
+test_tools.skip_if_missing("peg_generator")
+with test_tools.imports_under_tool("peg_generator"):
     from pegen.grammar_parser import GeneratedParser as GrammarParser
     from pegen.testutil import parse_string
     from pegen.first_sets import FirstSetCalculator
@@ -23,126 +23,167 @@ def test_alternatives(self) -> None:
             A: 'a' | '-'
             B: 'b' | '+'
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {
-            "A": {"'a'", "'-'"},
-            "B": {"'+'", "'b'"},
-            "expr": {"'+'", "'a'", "'b'", "'-'"},
-            "start": {"'+'", "'a'", "'b'", "'-'"},
-        })
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "A": {"'a'", "'-'"},
+                "B": {"'+'", "'b'"},
+                "expr": {"'+'", "'a'", "'b'", "'-'"},
+                "start": {"'+'", "'a'", "'b'", "'-'"},
+            },
+        )
 
     def test_optionals(self) -> None:
         grammar = """
             start: expr NEWLINE
             expr: ['a'] ['b'] 'c'
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {
-            "expr": {"'c'", "'a'", "'b'"},
-            "start": {"'c'", "'a'", "'b'"},
-        })
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "expr": {"'c'", "'a'", "'b'"},
+                "start": {"'c'", "'a'", "'b'"},
+            },
+        )
 
     def test_repeat_with_separator(self) -> None:
         grammar = """
         start: ','.thing+ NEWLINE
         thing: NUMBER
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {"thing": {"NUMBER"}, "start": {"NUMBER"}})
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"thing": {"NUMBER"}, "start": {"NUMBER"}},
+        )
 
     def test_optional_operator(self) -> None:
         grammar = """
         start: sum NEWLINE
         sum: (term)? 'b'
         term: NUMBER
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {
-            "term": {"NUMBER"},
-            "sum": {"NUMBER", "'b'"},
-            "start": {"'b'", "NUMBER"},
-        })
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "term": {"NUMBER"},
+                "sum": {"NUMBER", "'b'"},
+                "start": {"'b'", "NUMBER"},
+            },
+        )
 
     def test_optional_literal(self) -> None:
         grammar = """
         start: sum NEWLINE
         sum: '+' ? term
         term: NUMBER
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {
-            "term": {"NUMBER"},
-            "sum": {"'+'", "NUMBER"},
-            "start": {"'+'", "NUMBER"},
-        })
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "term": {"NUMBER"},
+                "sum": {"'+'", "NUMBER"},
+                "start": {"'+'", "NUMBER"},
+            },
+        )
 
     def test_optional_after(self) -> None:
         grammar = """
         start: term NEWLINE
         term: NUMBER ['+']
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"NUMBER"}})
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"term": {"NUMBER"}, "start": {"NUMBER"}},
+        )
 
     def test_optional_before(self) -> None:
         grammar = """
         start: term NEWLINE
         term: ['+'] NUMBER
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {"term": {"NUMBER", "'+'"}, "start": {"NUMBER", "'+'"}})
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"term": {"NUMBER", "'+'"}, "start": {"NUMBER", "'+'"}},
+        )
 
     def test_repeat_0(self) -> None:
         grammar = """
         start: thing* "+" NEWLINE
         thing: NUMBER
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {"thing": {"NUMBER"}, "start": {'"+"', "NUMBER"}})
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"thing": {"NUMBER"}, "start": {'"+"', "NUMBER"}},
+        )
 
     def test_repeat_0_with_group(self) -> None:
         grammar = """
         start: ('+' '-')* term NEWLINE
         term: NUMBER
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"'+'", "NUMBER"}})
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"term": {"NUMBER"}, "start": {"'+'", "NUMBER"}},
+        )
 
     def test_repeat_1(self) -> None:
         grammar = """
         start: thing+ '-' NEWLINE
         thing: NUMBER
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {"thing": {"NUMBER"}, "start": {"NUMBER"}})
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"thing": {"NUMBER"}, "start": {"NUMBER"}},
+        )
 
     def test_repeat_1_with_group(self) -> None:
         grammar = """
         start: ('+' term)+ term NEWLINE
         term: NUMBER
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"'+'"}})
+        self.assertEqual(
+            self.calculate_first_sets(grammar), {"term": {"NUMBER"}, "start": {"'+'"}}
+        )
 
     def test_gather(self) -> None:
         grammar = """
         start: ','.thing+ NEWLINE
         thing: NUMBER
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {"thing": {"NUMBER"}, "start": {"NUMBER"}})
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"thing": {"NUMBER"}, "start": {"NUMBER"}},
+        )
 
     def test_positive_lookahead(self) -> None:
         grammar = """
         start: expr NEWLINE
         expr: &'a' opt
         opt: 'a' | 'b' | 'c'
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {
-            "expr": {"'a'"},
-            "start": {"'a'"},
-            "opt": {"'b'", "'c'", "'a'"},
-        })
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "expr": {"'a'"},
+                "start": {"'a'"},
+                "opt": {"'b'", "'c'", "'a'"},
+            },
+        )
 
     def test_negative_lookahead(self) -> None:
         grammar = """
         start: expr NEWLINE
         expr: !'a' opt
         opt: 'a' | 'b' | 'c'
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {
-            "opt": {"'b'", "'a'", "'c'"},
-            "expr": {"'b'", "'c'"},
-            "start": {"'b'", "'c'"},
-        })
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "opt": {"'b'", "'a'", "'c'"},
+                "expr": {"'b'", "'c'"},
+                "start": {"'b'", "'c'"},
+            },
+        )
 
     def test_left_recursion(self) -> None:
         grammar = """
@@ -153,33 +194,42 @@ def test_left_recursion(self) -> None:
         bar: 'bar'
         baz: 'baz'
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {
-            "expr": {"NUMBER", "'-'"},
-            "term": {"NUMBER"},
-            "start": {"NUMBER", "'-'"},
-            "foo": {"'foo'"},
-            "bar": {"'bar'"},
-            "baz": {"'baz'"},
-        })
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "expr": {"NUMBER", "'-'"},
+                "term": {"NUMBER"},
+                "start": {"NUMBER", "'-'"},
+                "foo": {"'foo'"},
+                "bar": {"'bar'"},
+                "baz": {"'baz'"},
+            },
+        )
 
     def test_advance_left_recursion(self) -> None:
         grammar = """
         start: NUMBER | sign start
         sign: ['-']
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {"sign": {"'-'", ""}, "start": {"'-'", "NUMBER"}})
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"sign": {"'-'", ""}, "start": {"'-'", "NUMBER"}},
+        )
 
     def test_mutual_left_recursion(self) -> None:
         grammar = """
         start: foo 'E'
         foo: bar 'A' | 'B'
         bar: foo 'C' | 'D'
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {
-            "foo": {"'D'", "'B'"},
-            "bar": {"'D'"},
-            "start": {"'D'", "'B'"},
-        })
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "foo": {"'D'", "'B'"},
+                "bar": {"'D'"},
+                "start": {"'D'", "'B'"},
+            },
+        )
 
     def test_nasty_left_recursion(self) -> None:
         # TODO: Validate this
@@ -188,25 +238,33 @@ def test_nasty_left_recursion(self) -> None:
         target: maybe '+' | NAME
         maybe: maybe '-' | target
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {"maybe": set(), "target": {"NAME"}, "start": {"NAME"}})
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {"maybe": set(), "target": {"NAME"}, "start": {"NAME"}},
+        )
 
     def test_nullable_rule(self) -> None:
         grammar = """
         start: sign thing $
         sign: ['-']
         thing: NUMBER
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {
-            "sign": {"", "'-'"},
-            "thing": {"NUMBER"},
-            "start": {"NUMBER", "'-'"},
-        })
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "sign": {"", "'-'"},
+                "thing": {"NUMBER"},
+                "start": {"NUMBER", "'-'"},
+            },
+        )
 
     def test_epsilon_production_in_start_rule(self) -> None:
         grammar = """
         start: ['-'] $
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {"start": {"ENDMARKER", "'-'"}})
+        self.assertEqual(
+            self.calculate_first_sets(grammar), {"start": {"ENDMARKER", "'-'"}}
+        )
 
     def test_multiple_nullable_rules(self) -> None:
         grammar = """
@@ -216,10 +274,13 @@ def test_multiple_nullable_rules(self) -> None:
         other: '*'
         another: '/'
         """
-        self.assertEqual(self.calculate_first_sets(grammar), {
-            "sign": {"", "'-'"},
-            "thing": {"'+'", ""},
-            "start": {"'+'", "'-'", "'*'"},
-            "other": {"'*'"},
-            "another": {"'/'"},
-        })
+        self.assertEqual(
+            self.calculate_first_sets(grammar),
+            {
+                "sign": {"", "'-'"},
+                "thing": {"'+'", ""},
+                "start": {"'+'", "'-'", "'*'"},
+                "other": {"'*'"},
+                "another": {"'/'"},
+            },
+        )

Lib/test/test_peg_generator/test_grammar_validator.py

@@ -1,8 +1,8 @@
 import unittest
 from test import test_tools
 
-test_tools.skip_if_missing('peg_generator')
-with test_tools.imports_under_tool('peg_generator'):
+test_tools.skip_if_missing("peg_generator")
+with test_tools.imports_under_tool("peg_generator"):
     from pegen.grammar_parser import GeneratedParser as GrammarParser
     from pegen.validator import SubRuleValidator, ValidationError
     from pegen.testutil import parse_string