Merge pull request #180 from mammothb/feature/warn-when-failed-to-parse-frequency

Specify that frequency count must be 64-bit int

Author: mammothb

symspellpy/symspellpy.py

@@ -24,6 +24,7 @@
 import string
 import sys
 import unicodedata
+import warnings
 from collections import defaultdict
 from itertools import cycle
 from pathlib import Path
@@ -291,6 +292,8 @@ def load_bigram_dictionary(
         """Loads multiple dictionary entries from a file of word/frequency count
         pairs.
 
+        **NOTE**: Frequency count should be an integer that fits within 64 bits.
+
         **NOTE**: Merges with any dictionary data already loaded.
 
         Args:
@@ -323,6 +326,8 @@ def load_dictionary(
         """Loads multiple dictionary entries from a file of word/frequency count
         pairs.
 
+        **NOTE**: Frequency count should be an integer that fits within 64 bits.
+
         **NOTE**: Merges with any dictionary data already loaded.
 
         Args:
@@ -1090,6 +1095,7 @@ def _load_bigram_dictionary_stream(
                 continue
             count = helpers.try_parse_int64(parts[count_index])
             if count is None:
+                warnings.warn("Failed to parse frequency count as a 64 bit integer.")
                 continue
             key = (
                 f"{parts[term_index]} {parts[term_index + 1]}"
@@ -1128,6 +1134,7 @@ def _load_dictionary_stream(
                 continue
             count = helpers.try_parse_int64(parts[count_index])
             if count is None:
+                warnings.warn("Failed to parse frequency count as a 64 bit integer.")
                 continue
             key = parts[term_index]
             self.create_dictionary_entry(key, count)