Merge branch 'dev' into add-textaugment

Author: wannaphong

.github/workflows/pypi-publish.yml

@@ -22,10 +22,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+    - name: Publish a Python distribution to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}

docs/api/translate.rst

@@ -12,3 +12,9 @@ Modules
    :members: translate
 .. autoclass:: ThEnTranslator
    :members: translate
+.. autoclass:: ThZhTranslator
+   :members: translate
+.. autoclass:: ZhThTranslator
+   :members: translate
+.. autoclass:: Translate
+   :members:

docs/api/word_vector.rst

@@ -16,6 +16,8 @@ Modules
 .. autofunction:: most_similar_cosmul
 .. autofunction:: sentence_vectorizer
 .. autofunction:: similarity
+.. autoclass:: WordVector
+   :members:
 
 References
 ----------

pythainlp/corpus/default_db.json

@@ -1,7 +1,7 @@
 {
     "thainer": {
         "name": "thainer",
-        "latest_version": "1.5",
+        "latest_version": "1.5.1",
         "description": "Thai Named Entity Recognition",
         "long_description": "Thai Named Entity Recognition",
         "url": "https://github.com/wannaphong/thai-ner/",
@@ -11,8 +11,8 @@
         "author_email": "wannaphong@kkumail.com",
         "license": "cc-by-4.0",
         "versions": {
-            "1.5": {
-                "filename": "thainer_crf_1_5.model",
+            "1.5.1": {
+                "filename": "thainer_crf_1_5_1.model",
                 "download_url": "https://github.com/wannaphong/thai-ner/releases/download/1.5/thai-ner-1-5-newmm-lst20.crfsuite",
                 "md5": "-",
                 "pythainlp_version": ">=2.2.7"

pythainlp/corpus/thainer_crf_1_5.model

@@ -30,12 +30,15 @@ def clause_tokenize(doc: List[str]) -> List[List[str]]:
 
     :Example:
 
+    Clause tokenizer::
+
         from pythainlp.tokenize import clause_tokenize
 
         clause_tokenize(["ฉัน","นอน","และ","คุณ","เล่น","มือถือ","ส่วน","น้อง","เขียน","โปรแกรม"])
-        [['ฉัน', 'นอน'],
-        ['และ', 'คุณ', 'เล่น', 'มือถือ'],
-        ['ส่วน', 'น้อง', 'เขียน', 'โปรแกรม']]
+        # [['ฉัน', 'นอน'],
+        # ['และ', 'คุณ', 'เล่น', 'มือถือ'],
+        # ['ส่วน', 'น้อง', 'เขียน', 'โปรแกรม']]
+
     """
     if not doc or not isinstance(doc, str):
         return []
@@ -81,6 +84,8 @@ def word_tokenize(
         * *nercut* - Dictionary-based maximal matching word segmentation,
           constrained with Thai Character Cluster (TCC) boundaries,
           and combining tokens that are parts of the same named-entity.
+        * *sefr_cut* - wrapper for
+          `SEFR CUT <https://github.com/mrpeerat/SEFR_CUT>`_.,
 
     :Note:
         - The parameter **custom_dict** can be provided as an argument \
@@ -173,6 +178,10 @@ def word_tokenize(
     elif engine == "nercut":
         from pythainlp.tokenize.nercut import segment
 
+        segments = segment(text)
+    elif engine == "sefr_cut":
+        from pythainlp.tokenize.sefr_cut import segment
+
         segments = segment(text)
     else:
         raise ValueError(

pythainlp/corpus/thainer_crf_1_5_1.model

@@ -181,9 +181,9 @@ def segment(
         else:
             tokens = list(_onecut(sample, custom_dict))
             token_max_idx = 0
+            token_max_len = 0
             for i, token in enumerate(tokens):
-                token_max_len = 0
-                if len(token) > token_max_len:
+                if len(token) >= token_max_len:
                     token_max_len = len(token)
                     token_max_idx = i
 

pythainlp/tokenize/core.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+"""
+Wrapper for SEFR CUT Thai word segmentation. SEFR CUT is a
+Thai Word Segmentation Models using Stacked Ensemble.
+
+:See Also:
+    * `GitHub repository <https://github.com/mrpeerat/SEFR_CUT>`_
+"""
+from typing import List
+
+import sefr_cut
+
+DEFAULT_ENGINE = 'ws1000'
+sefr_cut.load_model(engine=DEFAULT_ENGINE)
+
+
+def segment(text: str, engine: str = 'ws1000') -> List[str]:
+    global DEFAULT_ENGINE
+    if not text or not isinstance(text, str):
+        return []
+    if engine != DEFAULT_ENGINE:
+        DEFAULT_ENGINE = engine
+        sefr_cut.load_model(engine=DEFAULT_ENGINE)
+    return sefr_cut.tokenize(text)[0]

pythainlp/tokenize/newmm.py

@@ -6,11 +6,20 @@
 __all__ = [
     "EnThTranslator",
     "ThEnTranslator",
-    "download_model_all"
+    "download_model_all",
+    "ThZhTranslator",
+    "ZhThTranslator",
+    "Translate"
 ]
 
-from pythainlp.translate.core import (
+from pythainlp.translate.core import Translate
+
+from pythainlp.translate.en_th import (
     EnThTranslator,
     ThEnTranslator,
     download_model_all,
 )
+from pythainlp.translate.zh_th import (
+    ThZhTranslator,
+    ZhThTranslator,
+)