Merge pull request #922 from ornariece/master

Fix #696 now providing the correct amount of placeholders

lark/load_grammar.py

@@ -345,12 +345,11 @@ def will_not_get_removed(sym):
                 return not sym.name.startswith('_')
             if isinstance(sym, Terminal):
                 return keep_all_tokens or not sym.filter_out
-            assert False
+            if sym is _EMPTY:
+                return False
+            assert False, sym
 
-        if any(rule.scan_values(will_not_get_removed)):
-            empty = _EMPTY
-        else:
-            empty = ST('expansion', [])
+        empty = ST('expansion', [_EMPTY] * len(list(rule.scan_values(will_not_get_removed))))
 
         return ST('expansions', [rule, empty])
 

tests/test_parser.py

@@ -2292,6 +2292,15 @@ def test_maybe_placeholders(self):
             self.assertEqual(p.parse("abba").children, ['a', None, 'b', 'b', 'a', None])
             self.assertEqual(p.parse("cbbbb").children, [None, 'c', 'b', 'b', 'b', 'b', None, None])
 
+            p = _Lark("""!start: ["a" "b" "c"] """, maybe_placeholders=True)
+            self.assertEqual(p.parse("").children, [None, None, None])
+            self.assertEqual(p.parse("abc").children, ['a', 'b', 'c'])
+
+            p = _Lark("""!start: ["a" ["b" "c"]] """, maybe_placeholders=True)
+            self.assertEqual(p.parse("").children, [None, None, None])
+            self.assertEqual(p.parse("a").children, ['a', None, None])
+            self.assertEqual(p.parse("abc").children, ['a', 'b', 'c'])
+
 
         def test_escaped_string(self):
             "Tests common.ESCAPED_STRING"