Merge pull request #50 from bact/main

Fix load_dict() output type + add unittest

nlpo3-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "nlpo3-python"
-version = "1.2.0"
+version = "1.2.1"
 edition = "2018"
 license = "Apache-2.0"
 authors = ["Thanathip Suntorntip Gorlph"]

nlpo3-python/nlpo3/__init__.py

@@ -1,25 +1,37 @@
-from typing import List
+# Python-binding for nlpO3, an natural language process library.
+#
+# Provides a tokenizer.
+#
+# Authors:
+# Thanathip Suntorntip
+# Arthit Suriyawongkul
+
+from pathlib import Path
+from typing import List, Tuple
 
 # import from .so (Rust)
 from ._nlpo3_python_backend import load_dict as rust_load_dict
 from ._nlpo3_python_backend import segment as rust_segment
 
 # TODO: load_dict from in-memory list of words
 
-def load_dict(file_path: str, dict_name: str) -> tuple[str, bool]:
+
+def load_dict(file_path: str, dict_name: str) -> Tuple[str, bool]:
  """Load dictionary from a file.
 
  Load a dictionary file into an in-memory dictionary collection,
  and assigned dict_name to it.
  *** This function does not override an existing dict name. ***
 
- :param file_path: Absolute path to a dictionary file
+ :param file_path: Path to a dictionary file
  :type file_path: str
  :param dict_name: A unique dictionary name, use for reference.
  :type dict_name: str
  :return tuple[human_readable_result_str, bool]
  """
- return rust_load_dict(file_path, dict_name)
+ path = Path(file_path).resolve()
+
+ return rust_load_dict(str(path), dict_name)
 
 
 def segment(
@@ -46,5 +58,9 @@ def segment(
  :return: List of tokens
  :rtype: List[str]
  """
+ if not text or not isinstance(text, str):
+ return []
+
  result = rust_segment(text, dict_name, safe, parallel)
+
  return result

nlpo3-python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "nlpo3"
-version = "1.2.0"
+version = "1.2.1"
 description = "Python binding for nlpO3 Thai language processing library in Rust"
 readme = "README.md"
 requires-python = ">=3.6"

nlpo3-python/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = nlpo3
-version = 1.2.0
+version = 1.2.1
 description = Python binding for nlpO3 Thai language processing library
 long_description =
  Python binding for nlpO3, a Thai natural language processing library in Rust.

nlpo3-python/setup.py

@@ -1,7 +1,8 @@
-from setuptools import setup
+from setuptools import find_packages, setup
 from setuptools_rust import Binding, RustExtension, Strip
 
 setup(
+ packages=find_packages(exclude=["notebooks", "tests"]),
  rust_extensions=[
  RustExtension(
  "nlpo3._nlpo3_python_backend",

nlpo3-python/src/lib.rs

@@ -1,3 +1,11 @@
+//! Python-binding for nlpO3, an natural language process library.
+//!
+//! Provides a tokenizer.
+//!
+//! Authors:
+//! Thanathip Suntorntip
+//! Arthit Suriyawongkul
+
 use ahash::AHashMap as HashMap;
 use lazy_static::lazy_static;
 use nlpo3::tokenizer;

nlpo3-python/tests/__init__.py

@@ -0,0 +1,12 @@
+"""
+Unit test
+"""
+import sys
+import unittest
+
+sys.path.append("../nlpo3")
+
+loader = unittest.TestLoader()
+testSuite = loader.discover("tests")
+testRunner = unittest.TextTestRunner(verbosity=1)
+testRunner.run(testSuite)

nlpo3-python/tests/data/test_dict.txt

@@ -0,0 +1,7 @@
+ค่า
+ค่าจ้าง
+ค่าจ้างเพื่อชีวิต
+ค่าแรง
+ค่ายทหาร
+คน
+ไข่