Merge pull request #168 from uiuc-focal-lab/bump

Fix IterGen parser issues

Author: shubhamugare

README.md

@@ -69,7 +69,7 @@ SynCode depends on HuggingFace [transformers](https://github.com/huggingface/tra
 
 | SynCode version | Required transformers version | Python version |
 | -------------- | ----------------------------- | -------------- |
-| `v0.4.6` (latest) | `v4.44.0`                    | 3.6 - 3.12     |
+| `v0.4.7` (latest) | `v4.44.0`                    | 3.6 - 3.12     |
 
 **Note:** Python 3.13 is not currently supported due to dependency constraints.
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "syncode"
-version="0.4.6"
+version="0.4.7"
 requires-python = ">=3.6,<3.13"
 description = "Grammar-guided code generation tool"
 readme = "README.md"

setup.py

@@ -18,7 +18,7 @@
 
 setuptools.setup(
     name="syncode",
-    version="0.4.6",
+    version="0.4.7",
     author="Shubham Ugare",
     author_email="shubhamugare@gmail.com",
     description="This package provides the tool for grammar augmented LLM generation.",

syncode/parsers/__init__.py

@@ -22,15 +22,15 @@ def create_parser(
         cache_filename = parser_cache_dir + f'{grammar}_{parser}_{grammar.hash()}_parser.pkl'
         os.makedirs(os.path.dirname(parser_cache_dir), exist_ok=True)
 
-        # First check if we should use the IGParser with symbol position map
-        if use_symbol_pos_map:
-            return IGParser(base_parser, **kwargs)
-
         if grammar.name == 'python':
             indenter = PythonIndenter()
 
         base_parser = create_base_parser(grammar, parser, indenter, cache_filename)
 
+        # First check if we should use the IGParser with symbol position map
+        if use_symbol_pos_map:
+            return IGParser(base_parser, **kwargs)
+        
         if grammar.name == 'python':
             return PythonIncrementalParser(base_parser, indenter, **kwargs)
         elif grammar.name == 'go':

syncode/parsers/itergen_parser.py

@@ -203,8 +203,8 @@ class IGParser(IncrementalParser):
     IterGen Parser extends IncrementalParser to add symbol position map functionality.
     This parser tracks positions of symbols in the code for code generation purposes.
     """
-    def __init__(self, base_parser, logger: Optional[common.Logger]=None, ignore_whitespace=False) -> None:
-        super().__init__(base_parser, logger, ignore_whitespace)
+    def __init__(self, base_parser, ignore_whitespace=False) -> None:
+        super().__init__(base_parser, ignore_whitespace=ignore_whitespace)
         # Current state mapping now includes symbol_pos_map
         self.cur_pos_to_parser_state: Dict[int, Tuple[Any, Any, Set, Set, Optional[list], list, Optional[SymbolPosMap]]] = {}
 
@@ -379,7 +379,7 @@ def get_acceptable_next_terminals(
 
         except lark.exceptions.UnexpectedToken as e:
             parse_incomplete = True
-            self._handle_parsing_error(lexer_tokens, token)
+            self._handle_parsing_error(lexer_tokens, token, e)
 
         # Compute current terminal string and return result
         remainder_state, current_term_str, final_terminal = self._get_remainder(