Added basic tests, added comment

lark/lexer.py

@@ -24,7 +24,7 @@
 ###{standalone
 from copy import copy
 
-try:
+try:  # For the standalone parser, we need to make sure that interegular is set to None to avoid NameErrors later on
     interegular
 except NameError:
     interegular = None
@@ -498,8 +498,8 @@ def __init__(self, conf: 'LexerConf', comparator=None) -> None:
                         assert a.priority == b.priority
                         # Mark this pair to not repeat warnings when multiple different BasicLexers see the same collision
                         comparator.mark(a, b)
-                        # leave it as a warning for the moment
 
+                        # leave it as a warning for the moment
                         # raise LexError("Collision between Terminals %s and %s" % (a.name, b.name))
                         logger.warning("Collision between Terminals %r and %r: %r" %
                                        (a.name, b.name, comparator.get_example_overlap(a, b)))
@@ -612,6 +612,7 @@ def __init__(self, conf: 'LexerConf', states: Dict[str, Collection[str]], always
             self.lexers[state] = lexer
 
         assert trad_conf.terminals is terminals
+        trad_conf.skip_validation = True  # We don't need to verify all terminals again
         self.root_lexer = BasicLexer(trad_conf, comparator)
 
     def lex(self, lexer_state: LexerState, parser_state: Any) -> Iterator[Token]:

test-requirements.txt

@@ -1,2 +1,3 @@
+interegular>=0.2.2
 Js2Py==0.68
 regex

tests/test_logger.py

@@ -1,13 +1,18 @@
 import logging
 from contextlib import contextmanager
 from lark import Lark, logger
-from unittest import TestCase, main
+from unittest import TestCase, main, skipIf
 
 try:
     from StringIO import StringIO
 except ImportError:
     from io import StringIO
 
+try:
+    import interegular
+except ImportError:
+    interegular = None
+
 @contextmanager
 def capture_log():
     stream = StringIO()
@@ -61,5 +66,37 @@ def test_loglevel_higher(self):
         # no log message
         self.assertEqual(len(log), 0)
 
+    @skipIf(interegular is None, "interegular is not installed, can't test regex collisions")
+    def test_regex_collision(self):
+        logger.setLevel(logging.WARNING)
+        collision_grammar = '''
+        start: A | B
+        A: /a+/
+        B: /(a|b)+/
+        '''
+        with capture_log() as log:
+            Lark(collision_grammar, parser='lalr')
+
+        log = log.getvalue()
+        # since there are conflicts between A and B
+        # symbols A and B should appear in the log message
+        self.assertIn("A", log)
+        self.assertIn("B", log)
+
+    @skipIf(interegular is None, "interegular is not installed, can't test regex collisions")
+    def test_regex_no_collision(self):
+        logger.setLevel(logging.WARNING)
+        collision_grammar = '''
+        start: A " " B
+        A: /a+/
+        B: /(a|b)+/
+        '''
+        with capture_log() as log:
+            Lark(collision_grammar, parser='lalr')
+
+        log = log.getvalue()
+        self.assertEqual(log, "")
+
+
 if __name__ == '__main__':
     main()