Fix fully-parsed validation check

Slight tweak to grammar for consistency
nightblade9 · Jan 21, 2018 · da8e2b0 · da8e2b0
1 parent 6c2c924
commit da8e2b0
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 19 deletions.
diff --git a/dragon/transpiler/lark/python3.g b/dragon/transpiler/lark/python3.g
@@ -173,7 +173,7 @@ _NEWLINE: ( /\r?\n[\t ]*/ | COMMENT )+
 %ignore /\\[\t \f]*\r?\n/   // LINE_CONT
 %ignore COMMENT
 
-STRING : /[ubf]?r?("(?!"").*?(?<!\\)(\\\\)*?"|'(?!'').*?(?<!\\)(\\\\)*?')/i
+STRING: /[ubf]?r?("(?!"").*?(?<!\\)(\\\\)*?"|'(?!'').*?(?<!\\)(\\\\)*?')/i
 LONG_STRING: /[ubf]?r?(""".*?(?<!\\)(\\\\)*?"""|'''.*?(?<!\\)(\\\\)*?''')/is
 
 DEC_NUMBER: /0|[1-9]\d*/i

diff --git a/dragon/transpiler/lark/validators/lark_validator.py b/dragon/transpiler/lark/validators/lark_validator.py
@@ -3,39 +3,36 @@
 
 class LarkValidator:
 
-    _TOKEN_REGEX = r"^([a-zA-Z0-9_\-]+):"
+    _TOKEN_REGEX = r"^([a-zA-Z0-9_\-\?!_\.]+):"
 
     def __init__(self, grammar_filename):
         self._grammar_tokens = self._extract_tokens(grammar_filename)
 
-    def is_fully_parsed(self, tree):
-        if isinstance(tree, str) and tree in self._tokens:
-            print("Found an unprocessed token: {}".format(tree))
-            return False
-        elif isinstance(tree, Tree):
-            if tree.data in self._tokens:
-                print("Found an unprocessed subtree: {}".format(tree.data))
-                return False
-            if len(tree.children):
-                for node in tree.children:
-                    if not self.is_fully_parsed(node):
-                        print("Found an unprocessed node: {}".format(node.data))
-                        return False
-
+    def is_fully_parsed(self, code):
+        for line in code:
+            for token in self._grammar_tokens:
+                if token in line:
+                    print("Found unparsed token {} in line {}".format(token, line))
+                    return False
+
         return True
 
     def _extract_tokens(self, grammar_filename):
         with open(grammar_filename, "rt") as f:
             grammar = f.read()
 
         lines = grammar.splitlines()
-        self._tokens = []
+        tokens = []
 
         for line in lines:
             match = re.search(LarkValidator._TOKEN_REGEX, line)
             if match:
-                token = match.group(1)
-                self._tokens.append(token)
+                token = match.group(1)                
+                token_name = token.replace("?", "").replace("!", "")
+                tokens.append(token_name)
+
+        return tokens
+
 
 def validate_class_definition(class_name, base_classes):
     if len(base_classes) > 1: