Remove obsolete and unused codes

- Remove pyicu uses in collation (already have _thkey() for sort key)
- Remove hunspell uses in spell (already have Peter Norvig's checker)
- Remove pythainlp.text.Text (nltk Text wrapper) (user can do this by themselve)
- Remove pylexto uses in tokenization

Author: bact

docs/pythainlp-dev-thai.md

@@ -57,7 +57,6 @@ engine คือ ระบบตัดคำ ปัจจุบันมี engi
 - longest - ใช้พจนานุกรม ด้วยวิธี Longest Matching
 - icu - เรียกใช้ตัวตัดคำจาก ICU ใช้พจนานุกรม (ความแม่นยำต่ำ)
 - wordcutpy - เรียกใช้ตัวตัดคำจาก [wordcutpy](https://github.com/veer66/wordcutpy) ใช้พจนานุกรม
-- pylexto - เรียกใช้ตัวตัดคำจาก LexTo ใช้พจนานุกรม ด้วยวิธี Longest Matching
 - deepcut - เรียกใช้ตัวตัดคำจาก [deepcut](https://github.com/rkcosmos/deepcut) ใช้การเรียนรู้ของเครื่อง
 
 คืนค่าเป็น ''list'' เช่น ['แมว', 'กิน']

examples/collation.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
 
-from pythainlp.collation import collation
+from pythainlp.collation import collate
 
-print(collation(["ไก่", "ไข่", "ก", "ฮา"]))  # ['ก', 'ไก่', 'ไข่', 'ฮา']
+print(collate(["ไก่", "ไข่", "ก", "ฮา"]))  # ['ก', 'ไก่', 'ไข่', 'ฮา']

examples/spell.py

@@ -8,7 +8,6 @@
 
 # spell checker from pythainlp.spell module (generic)
 print(spell("สี่เหลียม"))  # ['สี่เหลี่ยม']
-# print(spell("สี่เหลียม", engine="hunspell"))  # available in some Linux systems
 
 # spell checker from pythainlp.spell.pn module (specified algorithm - Peter Norvig's)
 print(pn_tnc_spell("เหลืยม"))

pythainlp/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 from pythainlp.change import texttoeng, texttothai
-from pythainlp.collation import collation
+from pythainlp.collation import collate
 from pythainlp.date import now
 from pythainlp.keywords import find_keyword
 from pythainlp.rank import rank

pythainlp/collation/__init__.py

@@ -1,41 +1,27 @@
 # -*- coding: utf-8 -*-
 """
-Thai collation (sort according to dictionary order)
-For Unicode collation, please refer to Unicode Common Locale Data Repository (CLDR)
-https://unicode.org/cldr/charts/latest/collation/th.html
+Thai collation (sort according to Thai dictionary order)
 """
 import re
 
-RE_TONE = re.compile(r"[็-์]")
-RE_LV_C = re.compile(r"([เ-ไ])([ก-ฮ])")
+_RE_TONE = re.compile(r"[็-์]")
+_RE_LV_C = re.compile(r"([เ-ไ])([ก-ฮ])")
 
-try:
-    import icu
 
-    thkey = icu.Collator.createInstance(icu.Locale("th_TH")).getSortKey
-except ImportError:
+def _thkey(word):
+    cv = _RE_TONE.sub("", word)  # remove tone
+    cv = _RE_LV_C.sub("\\2\\1", cv)  # switch lead vowel
+    tone = _RE_TONE.sub(" ", word)  # just tone
+    return cv + tone
 
-    def thkey(word):
-        cv = RE_TONE.sub("", word)  # remove tone
-        cv = RE_LV_C.sub("\\2\\1", cv)  # switch lead vowel
-        tone = RE_TONE.sub(" ", word)  # just tone
-        return cv + tone
 
-
-def collation(data):
+def collate(data):
     """
-    :param list data: a list of thai text
-    :return: a list of thai text, sorted alphabetically
+    :param list data: a list of strings
+    :return: a list of strings, sorted alphabetically, according to Thai rules
     **Example**::
         >>> from pythainlp.collation import *
         >>> collation(['ไก่', 'เป็ด', 'หมู', 'วัว'])
         ['ไก่', 'เป็ด', 'วัว', 'หมู']
     """
-    return sorted(data, key=thkey)
-
-
-if __name__ == "__main__":
-    a = collation(["ไก่", "ไข่", "ก", "ฮา"]) == ["ก", "ไก่", "ไข่", "ฮา"]
-    print(a)
-    print(collation(["หลาย", "หญิง"]) == ["หญิง", "หลาย"])
-    print(collation(["ไก่", "เป็ด", "หมู", "วัว"]) == ["ไก่", "เป็ด", "วัว", "หมู"])
+    return sorted(data, key=_thkey)

pythainlp/spell/__init__.py

@@ -3,18 +3,15 @@
 Spell checking
 """
 
+from .pn import spell as pn_spell
+
 
 def spell(word, engine="pn"):
     """
     :param str word: word to check spelling
     :param str engine:
         * pn - Peter Norvig's algorithm (default)
-        * hunspell - uses hunspell's algorithm, which should already exist in Linux
     :return: list of words
     """
-    if engine == "hunspell":
-        from .hunspell import spell as _spell
-    else:  # default, use "pn" engine
-        from .pn import spell as _spell
 
-    return _spell(word)
+    return pn_spell(word)

pythainlp/spell/hunspell.py

@@ -2,7 +2,6 @@
 """
 Unigram Part-Of-Speech Tagger
 """
-import json
 import os
 
 import dill

pythainlp/tag/old.py

@@ -24,7 +24,6 @@ def word_tokenize(text, engine="newmm", whitespaces=True):
         * longest - dictionary-based, Longest Matching
         * icu - wrapper for ICU, dictionary-based
         * wordcutpy - wrapper for wordcutpy, dictionary-based https://github.com/veer66/wordcutpy
-        * pylexto - wrapper for PyLexTo, dictionary-based, Longest Matching
         * deepcut - wrapper for deepcut, language-model-based https://github.com/rkcosmos/deepcut
         * ulmfit - use newmm engine with a specific dictionary for use with thai2vec
     :return: list of words, tokenized from the text
@@ -53,8 +52,6 @@ def segment(text):
         from .deepcut import segment
     elif engine == "wordcutpy":
         from .wordcutpy import segment
-    elif engine == "pylexto":
-        from .pylexto import segment
     elif engine == "mm" or engine == "multi_cut":
         from .multi_cut import segment
     else:  # default, use "newmm" engine