prevention: timeout every regex; fix: marker regex false pos on plotl…

…y plot html comments; prevention: timeout into_rich_tree;

Makefile

@@ -19,8 +19,7 @@ debug:
 	nodemon -L -V
 
 .PHONY: debug_command
-# debug_command: test
-debug_command: test_group7
+debug_command: test
 
 html_demo:
 	tree_plus https://en.wikipedia.org/wiki/Zero_ring
@@ -116,4 +115,7 @@ t4:
 	tree_plus tests/more_languages/group_todo
 
 t5:
-	tree_plus -h
+	tree_plus -h
+
+t6:
+	tree_plus -c -i group_todo tests/more_languages

README.md

@@ -39,7 +39,7 @@ Usage: tree_plus [OPTIONS] [PATHS]...
 
   Wrap patterns in quotes: -i "*.py" / -g "*.rs"
 
-  Example Invocations (These are not subcomands, you idiot):
+  Example Invocations:
 
           Show tree_plus_src and tests simultaneously
               > tree_plus tree_plus_src tests
@@ -852,6 +852,13 @@ make library_demo
 
 ## Moar Languages
 
+### Currently Tested Languages:
+<!-- t6-start -->
+```sh
+
+```
+<!-- t6-end -->
+### Without the -c "Concise" Flag:
 <!-- t2-start -->
 ```sh
 tree_plus -i group_todo tests/more_languages

tests/readme_updates/medium_README_source.md

@@ -1,5 +1,13 @@
 # Medium Readme
 
+<!-- t5-start -->
+```sh
+this 5th text should be removed
+```
+<!-- t5-end -->
+
+5th remaining text
+
 <!-- t1-start -->
 ```sh
 this 1st text should be removed
@@ -8,13 +16,13 @@ this 1st text should be removed
 
 1st remaining text
 
-<!-- t5-start -->
+<!-- t6-start -->
 ```sh
-this 5th text should be removed
+this 6th text should be removed
 ```
-<!-- t5-end -->
+<!-- t6-end -->
 
-5th remaining text
+6th remaining text
 
 <!-- t2-start -->
 ```sh

tests/test_deploy.py

@@ -123,7 +123,8 @@ def test_deploy_update_readme():
     assert "```\n<!-- t5-end -->" in final
     assert "this 5th text should be removed" not in final
     assert "Wrap patterns in quotes:" in final
-    # assert "within tests/more_languages" in final # not working b/c need to build new version
+    # # not working b/c need to build new version
+    # assert "within tests/more_languages" in final
     assert "5th remaining text" in final
 
 

tests/test_dotenv.py

@@ -1,5 +1,4 @@
 # tests/test_dotenv.py
-import pytest
 import os
 
 from tree_plus_src import debug_print

tests/test_engine.py

@@ -371,16 +371,16 @@ def test_engine_class_treeplus_stats():
     tp1 = engine.TreePlus()
     tp1_stats = tp1.stats()
     # assert tp1_stats == f"0 folders, 0 files, 0 lines, 0 tokens"
-    assert tp1_stats == f"0 folder(s), 0 file(s), 0 line(s), 0 token(s)"
+    assert tp1_stats == "0 folder(s), 0 file(s), 0 line(s), 0 token(s)"
     tp1 = engine.TreePlus(category=engine.Category.FOLDER)
     tp1_stats = tp1.stats()
     tp1.render()
-    assert tp1_stats == f"1 folder(s), 0 file(s), 0 line(s), 0 token(s)"
+    assert tp1_stats == "1 folder(s), 0 file(s), 0 line(s), 0 token(s)"
     tp2 = engine.TreePlus(category=engine.Category.FILE, line_count=1, token_count=1)
     tp1.subtrees.append(tp2)
     tp1_stats = tp1.stats()
     tp1.render()
-    assert tp1_stats == f"1 folder(s), 1 file(s), 1 line(s), 1 token(s)"
+    assert tp1_stats == "1 folder(s), 1 file(s), 1 line(s), 1 token(s)"
 
     # tp1.n_folders = 1
     # tp1.n_files = 1

tests/test_more_language_units.py

@@ -9,6 +9,7 @@
 from tree_plus_src import parse_file, replace_isabelle_symbols
 
 C_EXPECTATION = [
+    "TODO: technically we should build this module without AVX support",
     "struct Point",
     "struct Point getOrigin()",
     "float mul_two_floats(float x1, float x2)",

tests/test_units.py

@@ -78,7 +78,7 @@ def test_engine_parse_ignore_none():
         maybe_ignore_tuple=None, override=True
     )
     tree_plus.debug_print(f"{ignored_with_override=}")
-    assert ignored_with_override == None
+    assert ignored_with_override is None
 
 
 def test_engine_parse_ignore_one():
@@ -170,7 +170,7 @@ def test_units_parse_todo():
     content = open("tests/more_languages/group5/rust_todo_test.rs", "r").read()
     result = tree_plus.parse_markers(content)
     assert result == [
-        "TODO (Line 23): This todo tests parse_todo",
+        "TODO: This todo tests parse_todo",
     ]
 
 
@@ -182,9 +182,9 @@ def test_units_parse_todo():
 def test_units_parse_markers():
     results = tree_plus.parse_markers(bug_todo_note)
     assert results == [
-        "BUG (Line 1): This is a bug.",
-        "TODO (Line 2): Fix this soon.",
-        "NOTE (Line 3): Interesting observation.",
+        "BUG: This is a bug.",
+        "TODO: Fix this soon.",
+        "NOTE: Interesting observation.",
     ]
 
 
@@ -208,25 +208,6 @@ def test_units_token_counting_gpt4(file, expected):
     assert result == expected
 
 
-@pytest.mark.parametrize(
-    "file,expected",
-    [
-        (
-            "tests/path_to_test/file.py",
-            tree_plus.TokenLineCount(n_tokens=19, n_lines=3),
-        ),
-        (
-            "tests/path_to_test/empty.py",
-            tree_plus.TokenLineCount(n_tokens=0, n_lines=0),
-        ),
-    ],
-)
-def test_units_token_counting_gpt4(file, expected):
-    result = tree_plus.count_tokens_lines(file, tokenizer_name=TokenizerName.GPT4)
-    assert isinstance(result, tree_plus.TokenLineCount)
-    assert result == expected
-
-
 @pytest.mark.parametrize(
     "file,expected",
     [

tree_plus_cli.py

@@ -1,5 +1,5 @@
 # tree_plus_cli.py
-from typing import List, Optional, Union, Tuple
+from typing import Optional, Union, Tuple
 from time import perf_counter
 
 import click
@@ -11,6 +11,8 @@
     engine as tree_plus,
     TreePlus,
     DEFAULT_IGNORE,
+    regex_timeout,
+    set_regex_timeout,
     web,
 )
 from tree_plus_src.count_tokens_lines import TokenizerName
@@ -85,7 +87,7 @@
     "--yc",
     "--hn",
     is_flag=True,
-    help=f"Include ycombinator (False)",
+    help="Include ycombinator (False)",
     default=False,
 )
 @click.option(
@@ -99,7 +101,7 @@
     "--max-depth",
     "-m",
     "-M",
-    help="maximum number of steps (depth / level) from root (--yc mode only, default 3)",
+    help="max number of steps (depth / level) from root (--yc mode only, default 3)",
     default=3,
 )
 @click.option(
@@ -123,6 +125,12 @@
     default=None,
     type=str,
 )
+@click.option(
+    "--timeout",
+    help=f"regex timeout in seconds (optional, default {regex_timeout})",
+    default=None,
+    type=float,
+)
 @click.argument("paths", nargs=-1, type=click.UNPROCESSED)  # Accepts multiple arguments
 def main(
     # these are NON-MUTUALLY-EXCLUSIVE OPTIONS
@@ -142,12 +150,13 @@ def main(
     links: bool,
     tiktoken: bool,
     tokenizer_name: Optional[str],
+    timeout: Optional[float],
 ):
     """A `tree` util enhanced with tokens, lines, and components.
 
     Wrap patterns in quotes: -i "*.py" / -g "*.rs"
 
-    Example Invocations (These are not subcomands, you idiot):
+    Example Invocations:
 
         \b
         Show tree_plus_src and tests simultaneously
@@ -246,6 +255,9 @@ def main(
     else:
         raise ValueError(f"unsupported {tiktoken=} {tokenizer_name=}")
 
+    if timeout is not None and isinstance(timeout, float) and timeout > 0:
+        set_regex_timeout(timeout)
+
     root = tree_plus.from_seeds(
         _paths,
         maybe_ignore=ignore,

tree_plus_src/__init__.py

@@ -19,7 +19,13 @@
     amortize_globs,
 )
 
-from .parse_file import parse_file, parse_markers, extract_groups
+from .parse_file import (
+    regex_timeout,
+    parse_file,
+    parse_markers,
+    extract_groups,
+    set_regex_timeout,
+)
 from .count_tokens_lines import (
     TokenLineCount,
     TokenizerName,

tree_plus_src/count_tokens_lines.py

@@ -1,11 +1,14 @@
 # tree_plus_src/count_tokens_lines.py
 from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional, Union
 from enum import Enum
+
 import subprocess
-from typing import Optional, Union
-from pathlib import Path
 import os
 
+from rich.markdown import Markdown
+
 from tree_plus_src.debug import debug_print
 from tree_plus_src.parse_file import read_file
 
@@ -14,7 +17,7 @@
 
 
 # TODO: show off how well we parse_todo!
-@dataclass(frozen=True)
+@dataclass(frozen=True, slots=True)
 class TokenLineCount:
     n_tokens: int = 0
     n_lines: int = 0
@@ -161,9 +164,6 @@ def count_tokens_lines(
     return count
 
 
-from rich.markdown import Markdown
-
-
 def count_openai_tokens_lines_from_contents(
     contents: Union[str, Markdown],
     *,

tree_plus_src/deploy.py

@@ -6,6 +6,7 @@
 
 
 def extract(path: Optional[str] = None) -> str:
+    "get the content of the given path"
     assert path is not None
     assert os.path.exists(path)
     try:
@@ -18,6 +19,7 @@ def extract(path: Optional[str] = None) -> str:
 
 
 def load(content: Optional[str] = None, path: Optional[str] = None):
+    "write the content to the given path"
     assert content is not None
     assert isinstance(content, str)
     assert path is not None
@@ -82,7 +84,7 @@ def increment_version(
         print(f"increment_version extracted {major=} {minor=} {patch=}")
         new_patch = patch + 1
         new_line = f'__version__ = "{major}.{minor}.{new_patch}"'
-        print(f"increment_version new line:\n", new_line)
+        print("increment_version new line:\n", new_line)
         new_content_lines.append(new_line)
     new_content = "\n".join(new_content_lines)
     load(new_content, sink_path)
@@ -115,7 +117,7 @@ def replace_readme_section(
     assert marker is not None
     assert marker.startswith("t")
     assert int(marker[1]) > 0
-    assert int(marker[1]) < 6
+    assert int(marker[1]) < 7
     assert command is not None
     assert "make t" in command
 
@@ -175,6 +177,10 @@ def update_readme(source_path: Optional[str] = None, sink_path: Optional[str] =
         source_path=sink_path, sink_path=sink_path, marker="t5", command="make t5"
     )
     print("update_readme handled t5")
+    replace_readme_section(
+        source_path=sink_path, sink_path=sink_path, marker="t6", command="make t6"
+    )
+    print("update_readme handled t6")
     print("deploy::update_readme done")