⚡️ Speed up function sentence_count by 59%

CHANGELOG.md

@@ -1,13 +1,15 @@
 ## 0.18.14-dev0
 
 ### Enhancements
+- Speed up function sentence_count by 59% (codeflash)
 
 ### Features
 
 ### Fixes
 
 - **change short text language detection log to debug** reduce warning level log spamming
 
+
 ## 0.18.13
 
 ### Enhancements

unstructured/partition/text_type.py

@@ -219,15 +219,17 @@ def sentence_count(text: str, min_length: Optional[int] = None) -> int:
     sentences = sent_tokenize(text)
     count = 0
     for sentence in sentences:
-        sentence = remove_punctuation(sentence)
-        words = [word for word in word_tokenize(sentence) if word != "."]
-        if min_length and len(words) < min_length:
-            trace_logger.detail(  # type: ignore
-                f"Sentence does not exceed {min_length} word tokens, it will not count toward "
-                "sentence count.\n"
-                f"{sentence}",
-            )
-            continue
+        stripped = remove_punctuation(sentence)
+        # Fast token count after punctuation is removed: just split on whitespace
+        if min_length:
+            word_count = sum(1 for token in stripped.split() if token != ".")
+            if word_count < min_length:
+                trace_logger.detail(  # type: ignore
+                    f"Sentence does not exceed {min_length} word tokens, it will not count toward "
+                    "sentence count.\n"
+                    f"{stripped}",
+                )
+                continue
         count += 1
     return count