Terminology

3rd_party/CMakeLists.txt

@@ -31,6 +31,6 @@ get_directory_property(CMAKE_CXX_FLAGS DIRECTORY marian-dev DEFINITION CMAKE_CXX
 set(CMAKE_C_FLAGS ${CMAKE_C_FLAGS} PARENT_SCOPE)    
 set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} PARENT_SCOPE)    
 
-if(COMPILE_PYTHON)
-  add_subdirectory(pybind11)
-endif(COMPILE_PYTHON)
+#if(COMPILE_PYTHON)
+#  add_subdirectory(pybind11)
+#endif(COMPILE_PYTHON)

README.md

@@ -85,3 +85,55 @@ A short example of how to use the APIs is provided in `app/bergamot.cpp` file.
 ### Using WASM version
 
 Please follow the `README` inside the `wasm` folder of this repository that demonstrates how to use the translator in JavaScript.
+
+### Using python API
+
+Compile and install:
+```
+export CMAKE_BUILD_PARALLEL_LEVEL=8 # Use 8 cores to compile
+pip install wheel
+pip install .
+
+# Desktop app
+% bergamot-translator --help
+bergamot-translator interfance
+
+options:
+  -h, --help            show this help message and exit
+  --config CONFIG, -c CONFIG
+                        Model YML configuration input.
+  --num-workers NUM_WORKERS, -n NUM_WORKERS
+                        Number of CPU workers.
+  --logging LOGGING, -l LOGGING
+                        Set verbosity level of logging: trace, debug, info, warn, err(or), critical, off. Default is off
+  --cache-size CACHE_SIZE
+                        Cache size. 0 for caching is disabled
+  --terminology-tsv TERMINOLOGY_TSV, -t TERMINOLOGY_TSV
+                        Path to a terminology file TSV
+  --force-terminology, -f
+                        Force terminology to appear on the target side.
+  --path-to-input PATH_TO_INPUT, -i PATH_TO_INPUT
+                        Path to input file. Uses stdin if empty
+```
+Using the python interface
+```python
+from bergamot.translator import Translator
+print(Translator.__doc__)
+Bergamot translator interfacing with the C++ code.
+
+    Attributes:
+        num_workers        Number of parallel CPU workers.
+        cache:             Cache size. 0 to disable cache.
+        logging:           Log level: trace, debug, info, warn, err(or), critical, off. Default is off
+        terminology:       Path to a TSV terminology file
+        force_terminology  Force the terminology to appear on the target side. May affect translation quality negatively.
+
+        _config            Translation model config
+        _model:            Translation model
+        _responseOpts      What to include in the response (alignment, html restoration, etc..)
+        _service           The translation service
+
+translator = Translator("/path/to/model.npz.best-bleu.npz.decoder.brg.yml", terminology="/path/to/terminology.tsv")
+translator.translate(["text"])
+output
+```

bindings/python/CMakeLists.txt

@@ -1,3 +1,4 @@
+find_package(pybind11 REQUIRED)
 find_package(Python COMPONENTS Interpreter Development.Module REQUIRED)
 
 message("Using Python: " ${Python_EXECUTABLE})

bindings/python/bergamot.cpp

@@ -198,16 +198,22 @@ PYBIND11_MODULE(_bergamot, m) {
       .def("pivot", &ServicePyAdapter::pivot);
 
   py::class_<Service::Config>(m, "ServiceConfig")
-      .def(py::init<>([](size_t numWorkers, size_t cacheSize, std::string logging) {
+      .def(py::init<>([](size_t numWorkers, size_t cacheSize, std::string logging,
+      std::string pathToTerminologyFile, bool terminologyForce) {
              Service::Config config;
              config.numWorkers = numWorkers;
              config.cacheSize = cacheSize;
              config.logger.level = logging;
+             config.terminologyFile = pathToTerminologyFile;
+             config.terminologyForce = terminologyForce;
              return config;
            }),
-           py::arg("numWorkers") = 1, py::arg("cacheSize") = 0, py::arg("logLevel") = "off")
+           py::arg("numWorkers") = 1, py::arg("cacheSize") = 0, py::arg("logLevel") = "off",
+           py::arg("pathToTerminologyFile") = "", py::arg("terminologyForce") = false)
       .def_readwrite("numWorkers", &Service::Config::numWorkers)
-      .def_readwrite("cacheSize", &Service::Config::cacheSize);
+      .def_readwrite("cacheSize", &Service::Config::cacheSize)
+      .def_readwrite("pathToTerminologyFile", &Service::Config::terminologyFile)
+      .def_readwrite("terminologyForce", &Service::Config::terminologyForce);
 
   py::class_<_Model, std::shared_ptr<_Model>>(m, "TranslationModel");
 }

bindings/python/translator.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+import bergamot
+import argparse
+from sys import stdin
+from typing import List
+
+class Translator:
+    """Bergamot translator interfacing with the C++ code.
+
+    Attributes:
+        num_workers        Number of parallel CPU workers.
+        cache:             Cache size. 0 to disable cache.
+        logging:           Log level: trace, debug, info, warn, err(or), critical, off. Default is off
+        terminology:       Path to a TSV terminology file
+        force_terminology  Force the terminology to appear on the target side. May affect translation quality negatively.
+
+        _config            Translation model config
+        _model:            Translation model
+        _responseOpts      What to include in the response (alignment, html restoration, etc..)
+        _service           The translation service
+    """
+    num_workers: int
+    cache: int
+    logging: str
+    terminology: str
+    force_terminology: bool
+
+    _config: bergamot.ServiceConfig
+    _model: bergamot.TranslationModel
+    _responseOpts: bergamot.ResponseOptions
+    _service: bergamot.Service
+
+    def __init__(self, model_conifg_path: str, num_workers: int=1, cache: int=0, \
+                 logging="off", terminology: str="", force_terminology: bool=False):
+        """Initialises the translator class
+
+        :param model_conifg_path: Path to the configuration file for the translation model.
+        :param num_workers: Number of CPU workers.
+        :param cache: cache size. 0 means no cache.
+        :param logging: Log level: trace, debug, info, warn, err(or), critical, off.
+        :param terminology: Path to terminology file, TSV format
+        :param force_terminology: Force terminology to appear on the target side. May impact translation quality.
+        """
+        self.num_workers = num_workers
+        self.cache = cache
+        self.logging = logging
+        self.terminology = terminology
+        self.force_terminology = force_terminology
+
+        self._config = bergamot.ServiceConfig(self.num_workers, self.cache, self.logging, self.terminology, self.force_terminology)
+        self._service = bergamot.Service(self._config)
+        self._responseOpts = bergamot.ResponseOptions() # Default false for all, if we want to enable HTML later, from here
+        self._model = self._service.modelFromConfigPath(model_conifg_path)
+
+    def resetTerminology(self, terminology: str="", force_terminology: bool=False) -> None:
+        """Resets the terminology of the model
+        :param terminology: path to the terminology file.
+        :param force_terminology: force terminology
+        :return: None
+        """
+        self.terminology = terminology
+        self.force_terminology = force_terminology
+        self._config = bergamot.ServiceConfig(self.num_workers, self.cache, self.logging, self.terminology, self.force_terminology)
+        self._service = bergamot.Service(self._config)
+
+    def resetNumWorkers(self, num_workers) -> None:
+        """Resets the number of workers
+        :param num_workers: number of parallel CPU threads.
+        :return: None
+        """
+        self.num_workers = num_workers
+        self._config = bergamot.ServiceConfig(self.num_workers, self.cache, self.logging, self.terminology, self.force_terminology)
+        self._service = bergamot.Service(self._config)
+
+    def translate(self, sentences: List[str]) -> str:
+        """Translates a list of strings
+        :param sentences: A List of strings to be translated.
+        :return: Translation output.
+        """
+        responses = self._service.translate(self._model, bergamot.VectorString(sentences), self._responseOpts)
+        ret = ""
+        for response in responses:
+            ret = ret + response.target.text
+        return ret
+    #@TODO add async translate with futures
+
+def main():
+    parser = argparse.ArgumentParser(description="bergamot-translator interfance")
+    parser.add_argument("--config", '-c', required=True, type=str, help='Model YML configuration input.')
+    parser.add_argument("--num-workers", '-n', type=int, default=1, help='Number of CPU workers.')
+    parser.add_argument("--logging", '-l', type=str, default="off", help='Set verbosity level of logging: trace, debug, info, warn, err(or), critical, off. Default is off')
+    parser.add_argument("--cache-size", type=int, default=0, help='Cache size. 0 for caching is disabled')
+    parser.add_argument("--terminology-tsv", '-t', default="", type=str, help='Path to a terminology file TSV')
+    parser.add_argument("--force-terminology", '-f', action="store_true", help='Force terminology to appear on the target side.')
+    parser.add_argument("--path-to-input", '-i', default=None, type=str, help="Path to input file. Uses stdin if empty")
+    args = parser.parse_args()
+
+    translator = Translator(args.config, args.num_workers, args.cache_size, args.logging, args.terminology_tsv, args.force_terminology)
+
+    if args.path_to_input is not None:
+        with open(args.path_to_input, 'r', encoding='utf-8') as infile:
+            lines = infile.readlines()
+            print(translator.translate(lines))
+    else:
+        for line in stdin:
+            print(translator.translate([line.strip()]))
+
+if __name__ == '__main__':
+    main()

setup.py

@@ -195,8 +195,8 @@ def run(self):
 setup(
     name="bergamot",
     version=version,
-    author="Jerin Philip",
-    author_email="[email protected]",
+    author=["Jerin Philip", "Nikolay Bogoychev"],
+    author_email=["[email protected]", "[email protected]"],
     url="https://github.com/browsermt/bergamot-translator/",
     description="Translate text-content locally in your machine across langauges.",
     long_description=long_description,
@@ -209,10 +209,11 @@ def run(self):
     python_requires=">=3.6",
     packages=["bergamot"],
     package_dir={"bergamot": "bindings/python"},
-    install_requires=["requests", "pyyaml>=5.1", "appdirs"],
+    install_requires=["requests", "pyyaml>=5.1", "appdirs", "pybind11"],
     entry_points={
         "console_scripts": [
             "bergamot = bergamot.__main__:main",
+            "bergamot-translator = bergamot.translator:main"
         ],
     },
     # Classifiers help users find your project by categorizing it.

src/translator/service.cpp

@@ -6,12 +6,86 @@
 #include "batch.h"
 #include "byte_array_util.h"
 #include "definitions.h"
+#include <regex>
 
 namespace marian {
 namespace bergamot {
 
 namespace {
 
+  // Replacement_fn taken from https://stackoverflow.com/questions/3418231/replace-part-of-a-string-with-another-string
+  // Sue me.
+  size_t CountOccurrences(std::string_view s, std::string_view needle) {
+    size_t res = 0;
+    size_t pos = 0;
+    while ((pos = s.find(needle, pos)) != std::string_view::npos) {
+        ++res;
+        pos += needle.size();
+    }
+    return res;
+}
+
+std::string ReplaceNotLonger(std::string s, std::string_view what, std::string_view with) {
+    assert(what.size() >= with.size());
+    std::string_view::size_type wpos = 0;
+    std::string_view::size_type rpos = 0;
+    while (true) {
+        auto new_rpos = s.find(what, rpos);
+        if (new_rpos == std::string::npos) {
+            new_rpos = s.size();
+        }
+        auto n = new_rpos - rpos;
+        std::copy(s.begin() + rpos, s.begin() + new_rpos, s.begin() + wpos);
+        wpos += n;
+        rpos = new_rpos;
+        if (rpos == s.size()) {
+            break;
+        }
+        std::copy(with.begin(), with.end(), s.begin() + wpos);
+        wpos += with.size();
+        rpos += what.size();
+    }
+    s.resize(wpos);
+    return s;
+}
+
+std::string ReplaceLonger(std::string s, std::string_view what, std::string_view with) {
+    assert(what.size() < with.size());
+    auto occurrences = CountOccurrences(s, what);
+    auto rpos = s.size();
+    auto wpos = rpos + occurrences * (with.size() - what.size());
+    s.resize(wpos);
+
+    while (wpos != rpos) {
+        auto new_rpos = s.rfind(what, rpos - what.size());
+        if (new_rpos == std::string::npos) {
+            new_rpos = 0;
+        } else {
+            new_rpos += what.size();
+        }
+        auto n = rpos - new_rpos;
+        std::copy_backward(s.begin() + new_rpos, s.begin() + rpos, s.begin() + wpos);
+        wpos -= n;
+        rpos = new_rpos;
+        if (wpos == rpos) {
+            break;
+        }
+        std::copy_backward(with.begin(), with.end(), s.begin() + wpos);
+        wpos -= with.size();
+        rpos -= what.size();
+    }
+    return s;
+}
+
+std::string Replace(std::string s, std::string_view what, std::string_view with) {
+    assert(!what.empty());
+    if (what.size() >= with.size()) {
+        return ReplaceNotLonger(std::move(s), what, with);
+    }
+    return ReplaceLonger(std::move(s), what, with);
+}
+
+
 // Combines two responses with first.target == second.source mapping alignments etc accordingly.
 // There are several constraints which are matched by only the pivoting workflow in <>Service source, therefore this
 // function is not for external use and in a hidden namespace.
@@ -137,6 +211,43 @@ AsyncService::AsyncService(const AsyncService::Config &config)
       logger_(config.logger) {
   ABORT_IF(config_.numWorkers == 0, "Number of workers should be at least 1 in a threaded workflow");
   workers_.reserve(config_.numWorkers);
+  // Initiate terminology map if present
+  if (!config_.terminologyFile.empty()) {
+    // Create an input filestream
+    std::ifstream myFile(config_.terminologyFile);
+
+    // Make sure the file is open
+    if(!myFile.is_open()) throw std::runtime_error("Could not open file: " + config_.terminologyFile);
+    std::string line;
+    while(std::getline(myFile, line)) {
+        // Create a stringstream of the current line
+        std::stringstream ss(line);
+
+        std::string srcword;
+        std::string replacementword;
+        getline(ss, srcword, '\t');
+        getline(ss, replacementword, '\n');
+        // @TODO it seems like removing the tags forces the model to copy which is
+        // I guess just as good and more reliable. In that case we just don't tell the model
+        // what the original source is and it just has no choice BUT to generate the target.
+        if (!config_.terminologyForce) {
+          replacementword = srcword + " <tag0> " + replacementword + " </tag0> ";
+        }
+        this->terminologyMap_.insert({srcword, replacementword});
+    }
+
+    // Close file
+    myFile.close();
+
+    //Testing
+    if (config.logger.level == "debug") {
+      std::cerr << "Printing out terminology...:" << std::endl;
+      for (auto&& item : terminologyMap_) {
+        std::cerr << item.first << " " << item.second << std::endl;
+      }
+    }
+  }
+
   for (size_t cpuId = 0; cpuId < config_.numWorkers; cpuId++) {
     workers_.emplace_back([cpuId, this] {
       // Consumer thread main-loop. Note that this is an infinite-loop unless the monitor is explicitly told to
@@ -202,6 +313,11 @@ void AsyncService::pivot(std::shared_ptr<TranslationModel> first, std::shared_pt
 void AsyncService::translate(std::shared_ptr<TranslationModel> translationModel, std::string &&source,
                              CallbackType callback, const ResponseOptions &responseOptions) {
   // Producer thread, a call to this function adds new work items. If batches are available, notifies workers waiting.
+  // Tagging
+  for (auto&& teminologyPair : terminologyMap_) {
+    source = Replace(source, teminologyPair.first, teminologyPair.second);
+  }
+
   Ptr<HTML> html = std::make_shared<HTML>(std::move(source), responseOptions.HTML);
   auto internalCallback = [html, callback](Response &&response) {
     html->restore(response);