Skip to content

Commit

Permalink
Move PythonIndenter to lark.indenter; Fix reconstruct_python.py
Browse files Browse the repository at this point in the history
  • Loading branch information
erezsh committed Oct 13, 2021
1 parent ee664b5 commit ebb15f7
Show file tree
Hide file tree
Showing 5 changed files with 20 additions and 14 deletions.
10 changes: 1 addition & 9 deletions examples/advanced/python_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,9 @@
import glob, time

from lark import Lark
from lark.indenter import Indenter
from lark.indenter import PythonIndenter


class PythonIndenter(Indenter):
NL_type = '_NEWLINE'
OPEN_PAREN_types = ['LPAR', 'LSQB', 'LBRACE']
CLOSE_PAREN_types = ['RPAR', 'RSQB', 'RBRACE']
INDENT_type = '_INDENT'
DEDENT_type = '_DEDENT'
tab_len = 8

kwargs = dict(postlex=PythonIndenter(), start='file_input')

# Official Python grammar by Lark
Expand Down
4 changes: 2 additions & 2 deletions examples/advanced/qscintilla_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,15 +96,15 @@ def styleText(self, start, end):

try:
for token in self.lark.lex(text):
ws_len = token.pos_in_stream - last_pos
ws_len = token.start_pos - last_pos
if ws_len:
self.setStyling(ws_len, 0) # whitespace

token_len = len(bytearray(token, "utf-8"))
self.setStyling(
token_len, self.token_styles.get(token.type, 0))

last_pos = token.pos_in_stream + token_len
last_pos = token.start_pos + token_len
except Exception as e:
print(e)

Expand Down
9 changes: 7 additions & 2 deletions examples/advanced/reconstruct_python.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,15 @@
"""

from lark import Token
from lark import Token, Lark
from lark.reconstruct import Reconstructor
from lark.indenter import PythonIndenter

from python_parser import python_parser3

python_parser3 = Lark.open_from_package('lark', 'python.lark', ['grammars'],
parser='lalr', postlex=PythonIndenter(), start='file_input',
maybe_placeholders=False # Necessary for reconstructor
)

SPACE_AFTER = set(',+-*/~@<>="|:')
SPACE_BEFORE = (SPACE_AFTER - set(',:')) | set('\'')
Expand Down Expand Up @@ -63,6 +67,7 @@ def test():
output = python_reconstruct.reconstruct(tree, postproc)

tree_new = python_parser3.parse(output)
# assert tree.pretty() == tree_new.pretty()
assert tree == tree_new

print(output)
Expand Down
9 changes: 9 additions & 0 deletions lark/indenter.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,4 +100,13 @@ def DEDENT_type(self) -> str:
def tab_len(self) -> int:
raise NotImplementedError()


class PythonIndenter(Indenter):
NL_type = '_NEWLINE'
OPEN_PAREN_types = ['LPAR', 'LSQB', 'LBRACE']
CLOSE_PAREN_types = ['RPAR', 'RSQB', 'RBRACE']
INDENT_type = '_INDENT'
DEDENT_type = '_DEDENT'
tab_len = 8

###}
2 changes: 1 addition & 1 deletion tests/test_nearley/nearley

0 comments on commit ebb15f7

Please sign in to comment.