diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..43b90890 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 10 diff --git a/README.md b/README.md index 4fa8dd97..0c4fac3b 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ SynCode depends on HuggingFace [transformers](https://github.com/huggingface/tra | SynCode version | Required transformers version | Python version | | -------------- | ----------------------------- | -------------- | -| `v0.4.11` (latest) | `v4.51.0` | 3.6 - 3.12 | +| `v0.4.12` (latest) | `v4.51.0` | 3.6 - 3.12 | **Note:** Python 3.13 is not currently supported due to dependency constraints. diff --git a/pyproject.toml b/pyproject.toml index 89b68576..1d5c0baf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "syncode" -version="0.4.11" +version="0.4.12" requires-python = ">=3.6,<3.13" description = "Grammar-guided code generation tool" readme = "README.md" @@ -21,7 +21,7 @@ classifiers = [ dependencies = [ "fire", "interegular", - "regex==2023.8.8", + "regex==2024.11.6", "torch", "tqdm", "transformers==4.51.0", diff --git a/requirements.txt b/requirements.txt index aca1dee8..d9651c98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ accelerate fire interegular -regex==2023.8.8 +regex==2024.11.6 torch tqdm transformers==4.51.0; python_version < "3.13" diff --git a/setup.py b/setup.py index 9a78553d..f9b04013 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ requirements = [ "fire", "interegular", - "regex==2023.8.8", + "regex==2024.11.6", "torch", "tqdm", "transformers==4.51.0", @@ -18,7 +18,7 @@ setuptools.setup( name="syncode", - version="0.4.11", + version="0.4.12", author="Shubham Ugare", author_email="shubhamugare@gmail.com", description="This package provides the tool for grammar augmented LLM generation.", diff --git a/syncode/mask_store/mask_store.py b/syncode/mask_store/mask_store.py index 15ec1ef2..65cfdbc4 100644 --- a/syncode/mask_store/mask_store.py +++ b/syncode/mask_store/mask_store.py @@ -269,10 +269,10 @@ def _remove_left_whitespace( if (self._fsms.initial(fsm_state.terminal) == fsm_state or self._fsms.is_final(fsm_state)) and self._ignore_whitespace: if isinstance(remainder, bytes): # For bytes, use lstrip() to remove all leading whitespace - remainder = remainder.lstrip() + remainder = remainder.lstrip(b' ') elif isinstance(remainder, str): # For strings, use lstrip() to remove all leading whitespace - remainder = remainder.lstrip() + remainder = remainder.lstrip(' ') return remainder def _lookup_next_tokens_for_fsm_state(self, fsm_state: JointFSMState, next_terminal) -> torch.Tensor: diff --git a/syncode/parsers/itergen_parser.py b/syncode/parsers/itergen_parser.py index 3350c524..a7977dbf 100644 --- a/syncode/parsers/itergen_parser.py +++ b/syncode/parsers/itergen_parser.py @@ -102,7 +102,9 @@ def _update_symbol_pos_map_terminals(self, lexer_tokens: Iterable[Token], parsed # We don't add the last lexer token as it may change in the future # Essntially, we don't want IterGen to stop immediatelly after generating terminal which may extend in the future - start_idx -= 1 + if start_idx > 0: + start_idx -= 1 + end_idx = len(lexer_tokens)-1 for idx in range(start_idx, end_idx):