Decrease min chunk length from 30 --> 5 characters

Rutledge · Mar 23, 2023 · 5fabc57 · 5fabc57
1 parent 437656d
commit 5fabc57
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/services/chunks.py b/services/chunks.py
@@ -14,7 +14,7 @@
 # Constants
 CHUNK_SIZE = 200  # The target size of each text chunk in tokens
 MIN_CHUNK_SIZE_CHARS = 350  # The minimum size of each text chunk in characters
-MIN_CHUNK_LENGTH_TO_EMBED = 30  # Discard chunks shorter than this
+MIN_CHUNK_LENGTH_TO_EMBED = 5  # Discard chunks shorter than this
 EMBEDDINGS_BATCH_SIZE = 128  # The number of embeddings to request at a time
 MAX_NUM_CHUNKS = 10000  # The maximum number of chunks to generate from a text