Merge pull request #1150 from lark-parser/henryiii-henryiii/fix/types

Fix for PR #1149

.github/workflows/codecov.yml

@@ -10,16 +10,14 @@ jobs:
       OS: ${{ matrix.os }}
       PYTHON: '3.7'
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v3
     - name: Download submodules
-      run: |
-        git submodule update --init --recursive
-        git submodule sync -q
-        git submodule update --init
+      with:
+        submodules: recursive
     - name: Setup Python
-      uses: actions/setup-python@master
+      uses: actions/setup-python@v3
       with:
-        python-version: 3.7
+        python-version: "3.7"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/mypy.yml

@@ -4,16 +4,8 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
-      - name: Download submodules
-        run: git submodule update --init --recursive
-      - name: Set up Python
-        uses: actions/setup-python@v1
+      - uses: actions/checkout@v3
         with:
-          python-version: 3.8
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install mypy
+          submodules: recursive
       - name: Lint with mypy
-        run: mypy -p lark || true
+        run: pipx run tox -e type

.github/workflows/tests.yml

@@ -6,17 +6,14 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, 3.10.0-rc - 3.10, 3.11-dev, pypy3]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11-dev", "pypy-3.7"]
 
     steps:
-      - uses: actions/checkout@v2
-      - name: Download submodules
-        run: |
-          git submodule update --init --recursive
-          git submodule sync -q
-          git submodule update --init
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -25,4 +22,4 @@ jobs:
           pip install -r test-requirements.txt
       - name: Run tests
         run: |
-          python -m tests
+          python -m tests

lark/lark.py

@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 import sys, os, pickle, hashlib
 import tempfile
+import types
 from typing import (
     TypeVar, Type, List, Dict, Iterator, Callable, Union, Optional, Sequence,
     Tuple, Iterable, IO, Any, TYPE_CHECKING, Collection
@@ -27,9 +28,10 @@
 
 import re
 try:
-    import regex  # type: ignore
+    import regex
+    _has_regex = True
 except ImportError:
-    regex = None
+    _has_regex = False
 
 
 ###{standalone
@@ -253,11 +255,12 @@ class Lark(Serialize):
 
     def __init__(self, grammar: 'Union[Grammar, str, IO[str]]', **options) -> None:
         self.options = LarkOptions(options)
+        re_module: types.ModuleType
 
         # Set regex or re module
         use_regex = self.options.regex
         if use_regex:
-            if regex:
+            if _has_regex:
                 re_module = regex
             else:
                 raise ImportError('`regex` module must be installed if calling `Lark(regex=True)`.')
@@ -267,15 +270,15 @@ def __init__(self, grammar: 'Union[Grammar, str, IO[str]]', **options) -> None:
         # Some, but not all file-like objects have a 'name' attribute
         if self.options.source_path is None:
             try:
-                self.source_path = grammar.name
+                self.source_path = grammar.name  # type: ignore[union-attr]
             except AttributeError:
                 self.source_path = '<string>'
         else:
             self.source_path = self.options.source_path
 
         # Drain file-like objects to get their contents
         try:
-            read = grammar.read
+            read = grammar.read  # type: ignore[union-attr]
         except AttributeError:
             pass
         else:

lark/tree.py

@@ -5,6 +5,7 @@
 
 if TYPE_CHECKING:
     from .lexer import TerminalDef, Token
+    import rich
     if sys.version_info >= (3, 8):
         from typing import Literal
     else:

lark/tree_templates.py

@@ -9,14 +9,18 @@
 from lark import Tree, Transformer
 from lark.exceptions import MissingVariableError
 
+Branch = Union[Tree[str], str]
 TreeOrCode = Union[Tree[str], str]
+MatchResult = Dict[str, Tree]
 _TEMPLATE_MARKER = '$'
 
 
 class TemplateConf:
     """Template Configuration
 
     Allows customization for different uses of Template
+
+    parse() must return a Tree instance.
     """
 
     def __init__(self, parse=None):
@@ -49,15 +53,21 @@ def _get_tree(self, template: TreeOrCode) -> Tree[str]:
             assert self._parse
             template = self._parse(template)
 
-        assert isinstance(template, Tree)
+        if not isinstance(template, Tree):
+            raise TypeError("template parser must return a Tree instance")
+
         return template
 
     def __call__(self, template: Tree[str]) -> 'Template':
         return Template(template, conf=self)
 
-    def _match_tree_template(self, template: TreeOrCode, tree: TreeOrCode) -> Optional[Dict[str, TreeOrCode]]:
+    def _match_tree_template(self, template: TreeOrCode, tree: Branch) -> Optional[MatchResult]:
+        """Returns dict of {var: match} if found a match, else None
+        """
         template_var = self.test_var(template)
         if template_var:
+            if not isinstance(tree, Tree):
+                raise TypeError(f"Template variables can only match Tree instances. Not {tree!r}")
             return {template_var: tree}
 
         if isinstance(template, str):
@@ -100,7 +110,7 @@ def __default__(self, data, children, meta) -> Tree[str]:
 
 
 class Template:
-    """Represents a tree templates, tied to a specific configuration
+    """Represents a tree template, tied to a specific configuration
 
     A tree template is a tree that contains nodes that are template variables.
     Those variables will match any tree.
@@ -111,7 +121,7 @@ def __init__(self, tree: Tree[str], conf: TemplateConf = TemplateConf()):
         self.conf = conf
         self.tree = conf._get_tree(tree)
 
-    def match(self, tree: TreeOrCode) -> Optional[Dict[str, TreeOrCode]]:
+    def match(self, tree: TreeOrCode) -> Optional[MatchResult]:
         """Match a tree template to a tree.
 
         A tree template without variables will only match ``tree`` if it is equal to the template.
@@ -127,7 +137,7 @@ def match(self, tree: TreeOrCode) -> Optional[Dict[str, TreeOrCode]]:
         tree = self.conf._get_tree(tree)
         return self.conf._match_tree_template(self.tree, tree)
 
-    def search(self, tree: TreeOrCode) -> Iterator[Tuple[Tree[str], Dict[str, TreeOrCode]]]:
+    def search(self, tree: TreeOrCode) -> Iterator[Tuple[Tree[str], MatchResult]]:
         """Search for all occurances of the tree template inside ``tree``.
         """
         tree = self.conf._get_tree(tree)

lark/utils.py

@@ -109,9 +109,10 @@ def deserialize(cls, data, namespace, memo):
 
 
 try:
-    import regex  # type: ignore
+    import regex
+    _has_regex = True
 except ImportError:
-    regex = None
+    _has_regex = False
 
 if sys.version_info >= (3, 11):
     import re._parser as sre_parse
@@ -123,7 +124,7 @@ def deserialize(cls, data, namespace, memo):
 categ_pattern = re.compile(r'\\p{[A-Za-z_]+}')
 
 def get_regexp_width(expr):
-    if regex:
+    if _has_regex:
         # Since `sre_parse` cannot deal with Unicode categories of the form `\p{Mn}`, we replace these with
         # a simple letter, which makes no difference as we are only trying to get the possible lengths of the regex
         # match here below.
@@ -135,7 +136,7 @@ def get_regexp_width(expr):
     try:
         return [int(x) for x in sre_parse.parse(regexp_final).getwidth()]
     except sre_constants.error:
-        if not regex:
+        if not _has_regex:
             raise ValueError(expr)
         else:
             # sre_parse does not support the new features in regex. To not completely fail in that case,
@@ -223,15 +224,16 @@ def combine_alternatives(lists):
 
 try:
     import atomicwrites
+    _has_atomicwrites = True
 except ImportError:
-    atomicwrites = None  # type: ignore[assigment]
+    _has_atomicwrites = False
 
 class FS:
     exists = staticmethod(os.path.exists)
 
     @staticmethod
     def open(name, mode="r", **kwargs):
-        if atomicwrites and "w" in mode:
+        if _has_atomicwrites and "w" in mode:
             return atomicwrites.atomic_write(name, mode=mode, overwrite=True, **kwargs)
         else:
             return open(name, mode, **kwargs)

pyproject.toml

@@ -0,0 +1,18 @@
+[build-system]
+requires = ["setuptools>=42"]
+build-backend = "setuptools.build_meta"
+
+
+[tool.mypy]
+files = "lark"
+python_version = "3.6"
+show_error_codes = true
+enable_error_code = ["ignore-without-code"]
+exclude = [
+  "^lark/__pyinstaller",
+]
+
+# You can disable imports or control per-module/file settings here
+[[tool.mypy.overrides]]
+module = [ "js2py" ]
+ignore_missing_imports = true

tests/test_tree_templates.py

@@ -87,6 +87,12 @@ def test_template_match__default_conf_match_same_tree__empty_dictionary(self):
 
         self.assertEqual({}, template.match(SOME_NON_TEMPLATE_TREE))
 
+    def test_template_match__only_tree(self):
+        "This test might become irrelevant in the future"
+        template_tree = Tree('bar', [Tree("var", children=["$foo"])])
+        template = Template(template_tree)
+        self.assertRaises(TypeError, template.match, Tree('bar', ['BAD']))
+
 
 class TestTreeTemplatesTemplate(unittest.TestCase):
     parser = Lark(SOME_TEMPLATING_GRAMMAR)

tox.ini

@@ -1,16 +1,30 @@
 [tox]
-envlist = py36, py37, py38, py39, pypy, pypy3
-skip_missing_interpreters=true
+envlist = py36, py37, py38, py39, py310, pypy3, type
+skip_missing_interpreters = true
 
 [testenv]
 whitelist_externals = git
 deps =
     -rtest-requirements.txt
+passenv =
+    TERM
 
 # to always force recreation and avoid unexpected side effects
-recreate=True
+recreate = True
 
-commands=
+commands =
     git submodule sync -q
     git submodule update --init
     python -m tests {posargs}
+
+[testenv:type]
+description = run type check on code base
+skip_install = true
+recreate = false
+deps =
+    mypy==0.950
+    types-atomicwrites
+    types-regex
+    rich
+commands =
+    mypy