Merge pull request #987 from bact/add-nlpo3-dep

Add nlpo3 to compact install

Author: bact

docker_requirements.txt

@@ -11,7 +11,7 @@ fastcoref==2.1.6
 gensim==4.3.2
 h5py==3.1.0
 khanaa==0.0.6
-nlpo3==1.3.0
+nlpo3==1.3.1
 nltk==3.6.6
 numpy==1.22.*
 OSKut==1.3

pythainlp/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # SPDX-FileCopyrightText: 2016-2024 PyThaiNLP Project
 # SPDX-License-Identifier: Apache-2.0
-__version__ = "5.0.4"
+__version__ = "5.0.5-dev"
 
 thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ"  # 44 chars
 

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 5.0.4
+current_version = 5.0.5-dev
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<build>\d+))?

setup.py

@@ -67,7 +67,7 @@
     "ipa": ["epitran>=1.1"],
     "ml": ["numpy>=1.22", "torch>=1.0.0"],
     "mt5": ["sentencepiece>=0.1.91", "transformers>=4.6.0"],
-    "nlpo3": ["nlpo3>=1.3.0"],
+    "nlpo3": ["nlpo3>=1.3.1"],
     "onnx": ["numpy>=1.22", "onnxruntime>=1.10.0", "sentencepiece>=0.1.91"],
     "oskut": ["oskut>=1.3"],
     "sefr_cut": ["sefr_cut>=1.1"],
@@ -103,6 +103,7 @@
     # Compact dependencies, this one matches requirements.txt
     "compact": [
         "PyYAML>=5.4.1",
+        "nlpo3>=1.3.1",
         "numpy>=1.22",
         "pyicu>=2.3",
         "python-crfsuite>=0.9.7",
@@ -119,7 +120,7 @@
         "fastcoref>=2.1.5",
         "gensim>=4.0.0",
         "khamyo>=0.2.0",
-        "nlpo3>=1.3.0",
+        "nlpo3>=1.3.1",
         "nltk>=3.3",
         "numpy>=1.22",
         "onnxruntime>=1.10.0",
@@ -148,7 +149,7 @@
 
 setup(
     name="pythainlp",
-    version="5.0.4",
+    version="5.0.5-dev",
     description="Thai Natural Language Processing library",
     long_description=LONG_DESC,
     long_description_content_type="text/markdown",

tests/README.md

@@ -16,7 +16,7 @@ Tests are categorized into three groups: core, compact, and extra.
 - Run `unittest tests.compact`
 - Test a limited set of functionalities that rely on a stable and small subset
   of optional dependencies specified in `requirements.txt`.
-- These dependencies are `PyYAML`, `numpy`, `pyicu`,
+- These dependencies are `PyYAML`, `nlpo3`, `numpy`, `pyicu`,
   `python-crfsuite`, and `requests`.
 - Test with the latest two stable Python versions.
 

tests/compact/testc_tokenize.py

@@ -88,3 +88,9 @@ def test_icu(self):
 
     def test_word_tokenize_icu(self):
         self.assertIsNotNone(word_tokenize(TEXT_1, engine="icu"))
+
+
+class WordTokenizeNlpO3TestCase(unittest.TestCase):
+    def test_word_tokenize_nlpo3(self):
+        self.assertIsNotNone(word_tokenize(TEXT_1, engine="nlpo3"))
+        self.assertEqual(word_tokenize("การ์", engine="nlpo3"), ["การ์"])

tests/extra/testx_tokenize.py

@@ -306,11 +306,6 @@ def test_nercut(self):
         self.assertIsNotNone(word_tokenize("ทดสอบ", engine="nercut"))
 
 
-class WordTokenizeNlpO3TestCase(unittest.TestCase):
-    def test_word_tokenize_nlpo3(self):
-        self.assertIsNotNone(word_tokenize(TEXT_1, engine="nlpo3"))
-
-
 class WordTokenizeOSKutTestCase(unittest.TestCase):
     def test_word_tokenize_oskut(self):
         self.assertIsNotNone(word_tokenize(TEXT_1, engine="oskut"))