Peg parser

.asv/results/benchmarks.json

@@ -4,20 +4,26 @@
         "name": "benchmarks.MemSuite.mem_parser_init",
         "param_names": [],
         "params": [],
-        "timeout": 60.0,
         "type": "memory",
         "unit": "bytes",
         "version": "fef8294dd8f9528bab51728ace954f992f353699e8c20ddba9cb190744896302"
     },
-    "benchmarks.PeakMemSuite.peakmem_parser_init": {
-        "code": "class PeakMemSuite:\n    def peakmem_parser_init(self):\n        from xonsh_parser.parser import get_parser_cls\n    \n        parser = get_parser_cls()()\n        parser.parse(\"ls -alh\")\n\n    def setup(self):\n        from xonsh_parser.parser import write_parser_table\n        write_parser_table()",
-        "name": "benchmarks.PeakMemSuite.peakmem_parser_init",
-        "param_names": [],
-        "params": [],
-        "timeout": 60.0,
+    "benchmarks.PeakMemSuite.peakmem_parser_init_": {
+        "code": "class PeakMemSuite:\n    def peakmem_parser_init_(self, f):\n    \n        from xonsh_parser.parser import get_parser_cls\n    \n        parser = get_parser_cls()(parser_table=Path(f))\n        parser.parse(\"ls -alh\")\n\n    def setup(self, f):\n        from xonsh_parser.parser import write_parser_table\n        write_parser_table(output_path=f)",
+        "name": "benchmarks.PeakMemSuite.peakmem_parser_init_",
+        "param_names": [
+            "param1"
+        ],
+        "params": [
+            [
+                "'/tmp/xonsh-lr-table.pickle'",
+                "'/tmp/xonsh-lr-table.py'",
+                "'/tmp/xonsh-lr-table.jsonl'"
+            ]
+        ],
         "type": "peakmemory",
         "unit": "bytes",
-        "version": "5395f5577f2ddc095088b4765b106fbc215e28fda9ae9433dcb13f718c8e3272"
+        "version": "d76e635d01b229185dffca78bc73e779e1a383322bf2cb3d30fbe427bc289db4"
     },
     "benchmarks.TimeSuite.time_parser_init": {
         "code": "class TimeSuite:\n    def time_parser_init(self):\n        from xonsh_parser.parser import get_parser_cls\n    \n        parser = get_parser_cls()()\n        parser.parse(\"ls -alh\")\n\n    def setup(self):\n        from xonsh_parser.parser import write_parser_table\n        write_parser_table()",
@@ -29,21 +35,28 @@
         "repeat": 0,
         "rounds": 2,
         "sample_time": 0.01,
-        "timeout": 60.0,
         "type": "time",
         "unit": "seconds",
         "version": "e35e8735a755146c49df0237cdf07c88a943702d6e364e50d29f2aa842bb1990",
         "warmup_time": -1
     },
     "benchmarks.TrackLrParserSize.track_lr_parser_size": {
-        "code": "class TrackLrParserSize:\n    def track_lr_parser_size(self):\n        from pympler import asizeof\n    \n        from xonsh_parser.parser import get_parser_cls\n    \n        parser = get_parser_cls()()\n        return asizeof.asizeof(parser.parser)\n\n    def setup(self):\n        from xonsh_parser.parser import write_parser_table\n        write_parser_table()",
+        "code": "class TrackLrParserSize:\n    def track_lr_parser_size(self, f):\n        from pympler import asizeof\n    \n        from xonsh_parser.parser import get_parser_cls\n    \n        parser = get_parser_cls()(parser_table=Path(f))\n        return asizeof.asizeof(parser.parser)\n\n    def setup(self, f):\n        from xonsh_parser.parser import write_parser_table\n        write_parser_table(output_path=f)",
         "name": "benchmarks.TrackLrParserSize.track_lr_parser_size",
-        "param_names": [],
-        "params": [],
-        "timeout": 60.0,
+        "param_names": [
+            "param1"
+        ],
+        "params": [
+            [
+                "'/tmp/xonsh-lr-table.pickle'",
+                "'/tmp/xonsh-lr-table.py'",
+                "'/tmp/xonsh-lr-table.jsonl'",
+                "'/tmp/xonsh-lr-table.cpickle'"
+            ]
+        ],
         "type": "track",
         "unit": "bytes",
-        "version": "5c2ee1d6c02c954f783f168723747a697bd1226feb95ad3c24dfa9eed58372e7"
+        "version": "dcdc63d51d9781e39b68aaff072ff921d29bcf2b78281dccc3826b520ca78a2d"
     },
     "version": 2
 }

.github/workflows/build.yml

@@ -1,33 +1,46 @@
 name: Build
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  contents: read
 
 jobs:
   test:
-
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        python_version: ['3.9']
-
+        os:
+          - ubuntu-latest
+          - macOS-latest
+          - windows-latest
+        python-version:
+          - "3.10"
+          - "3.11"
+          - "3.12"
+          - "3.13-dev"
+    name: Test Python ${{ matrix.python-version }} ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python_version }}
+        cache: 'pip'
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install hatch
-        hatch env create
-    - name: Lint and typecheck
-      run: |
-        hatch run lint-check
+        pip install -U pip
+        pip install -e ".[test]"
     - name: Test
-      run: |
-        hatch run test-cov-xml
-    - uses: codecov/codecov-action@v3
+      run: pytest --cov=peg_parser/parser tests/ --cov-report=term-missing --cov-report=xml
+    - uses: codecov/codecov-action@v4.0.1
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         fail_ci_if_error: true

.gitignore

@@ -125,5 +125,6 @@ pdm.lock
 # ply parser debug logs
 *.out
 .pdm-python
+.task/
 
-.history/
+.testmondata*

.pre-commit-config.yaml

@@ -1,29 +1,23 @@
 repos:
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.0.263"
+    rev: "v0.4.2"
     hooks:
       - id: ruff
-        args: ["--fix"]
-
-  - repo: https://github.com/ambv/black
-    rev: "23.3.0"
-    hooks:
-      - id: black
+        args: [ ., --fix, --exit-non-zero-on-fix ]
+        pass_filenames: false
+      - id: ruff-format
+        args: [.]
         pass_filenames: false
-        args:
-          - xonsh_parser
-          - tests
-        language_version: python3.8
-
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.2.0" # Use the sha / tag you want to point at
+    rev: "v1.10.0" # Use the sha / tag you want to point at
     hooks:
       - id: mypy
         pass_filenames: false
+        args: [ "--config-file=pyproject.toml"]
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: "v4.4.0"
+    rev: "v4.6.0"
     hooks:
       - id: trailing-whitespace
       - id: check-yaml
@@ -33,7 +27,7 @@ repos:
       - id: check-added-large-files
 
   - repo: https://github.com/compilerla/conventional-pre-commit
-    rev: "v2.2.0"
+    rev: "v3.2.0"
     hooks:
       - id: conventional-pre-commit
         stages: [commit-msg]

README.md

@@ -27,7 +27,10 @@ We use [Hatch](https://hatch.pypa.io/latest/install/) to manage the development
 You can run all the tests with:
 
 ```bash
-hatch run test
+task test
+
+# to watch for changes and run tests
+task test --watch -- -x --ff
 ```
 
 ### Format the code

Taskfile.yml

@@ -1,18 +1,37 @@
-# https://taskfile.dev
-
 version: '3'
+tasks:
+  generate:
+    cmds:
+      - python3 peg_parser/tasks/generate_parser.py
+    generates:
+      - peg_parser/parser/parser.py
+    sources:
+      - peg_parser/parser/xonsh.gram
+      - peg_parser/parser/*.py
+      - pegen/*.py
+
+  profile:
+    cmds:
+      - python peg_parser/tasks/profile_mem.py tee "logs/xonsh-parser-$(date "+%Y%m%d-%H%M%S").log"
 
-vars:
-  GREETING: Hello, World!
+  ply-add:
+    cmds:
+      - git subtree add --prefix=ply --squash
 
-tasks:
-  add-ply-subtree:
+  pegen-add:
     cmds:
-      - git subtree add --prefix=ply --squash https://github.com/dabeaz/ply.git master
-  add-pegen-subtree:
+      - git fetch https://github.com/we-like-parsers/pegen.git main:tmp-pegen-main --no-tags --depth 1
+      - git show tmp-pegen-main:data/python.gram > .local/python.gram
+      - git read-tree --prefix=pegen -u tmp-pegen-main:src/pegen
+
+  test:
+    deps:
+      - generate
     cmds:
-      - git subtree add --prefix=pegen --squash
+      - python -m pytest {{.CLI_ARGS}}
+    sources:
+      - '**/*.py'
 
-  asv:
+  wtest:
     cmds:
-      - asv run
+      - watchexec -e py,gram --clear -- task test -- --ff -x -vv --testmon

benchmarks/benchmarks.py

@@ -1,5 +1,6 @@
 # Write the benchmarking functions here.
 # See "Writing benchmarks" in the asv docs for more information.
+from pathlib import Path
 
 
 class TimeSuite:
@@ -28,28 +29,32 @@ def mem_parser_init(self):
 
 
 class PeakMemSuite:
-    def setup(self):
+    params = ["/tmp/xonsh-lr-table.pickle", "/tmp/xonsh-lr-table.py", "/tmp/xonsh-lr-table.jsonl"]
+    def setup(self, f):
         from xonsh_parser.parser import write_parser_table
-        write_parser_table()
+        write_parser_table(output_path=f)
+
+    def peakmem_parser_init_(self, f):
 
-    def peakmem_parser_init(self):
         from xonsh_parser.parser import get_parser_cls
 
-        parser = get_parser_cls()()
+        parser = get_parser_cls()(parser_table=Path(f))
         parser.parse("ls -alh")
 
 
 class TrackLrParserSize:
     unit = "bytes"
+    params = ["/tmp/xonsh-lr-table.pickle", "/tmp/xonsh-lr-table.py", "/tmp/xonsh-lr-table.jsonl",
+              "/tmp/xonsh-lr-table.cpickle"]
 
-    def setup(self):
+    def setup(self, f):
         from xonsh_parser.parser import write_parser_table
-        write_parser_table()
+        write_parser_table(output_path=f)
 
-    def track_lr_parser_size(self):
+    def track_lr_parser_size(self, f):
         from pympler import asizeof
 
         from xonsh_parser.parser import get_parser_cls
 
-        parser = get_parser_cls()()
+        parser = get_parser_cls()(parser_table=Path(f))
         return asizeof.asizeof(parser.parser)

experiments.md

@@ -53,3 +53,46 @@ PosixPath('/tmp/v1-bytes.pickle')
 # tried using marshal
 
 - there was not much difference with pickle
+
+# fixing reduce/reduce conflicts
+
+1. initial sizes
+
+```
+asizeof.asizeof(productions)=260KiB
+asizeof.asizeof(actions)=6.18MiB
+asizeof.asizeof(gotos)=560KiB
+```
+
+| type   | file-size |
+|--------|-----------|
+| pickle | 975.55KiB |
+| py     | 1.61 MiB  |
+| jsonl  | 1.61 MiB  |
+
+2. merge overridden actions to base class - no change in sizes
+
+```
+_object_size(productions)='260.96 KiB'
+_object_size(actions)='6.18 MiB'
+_object_size(gotos)='560.97 KiB'
+```
+
+3. peak memory usage using lr-table.py type
+
+benchmarks.PeakMemSuite.peakmem_parser_init_                                                                                 ok
+[75.00%] ··· ============================ =======
+                        param1
+             ---------------------------- -------
+              /tmp/xonsh-lr-table.pickle   31.5M
+                /tmp/xonsh-lr-table.py      215M
+              /tmp/xonsh-lr-table.jsonl    33.5M
+             ============================ =======
+ benchmarks.TrackLrParserSize.track_lr_parser_size                                                                    1/4 failed
+[100.00%] ··· ============================= ========
+                          param1
+              ----------------------------- --------
+                /tmp/xonsh-lr-table.pickle   7.54M
+                  /tmp/xonsh-lr-table.py      4.8M
+                /tmp/xonsh-lr-table.jsonl    7.53M
+               /tmp/xonsh-lr-table.cpickle   7.54M

peg_parser/parser/.gitignore

@@ -0,0 +1,2 @@
+# generated files
+parser.py