limit number of in-memory Simplemma dictionaries to at most 5

annif/analyzer/simplemma.py

@@ -3,6 +3,8 @@
 
 import simplemma
 
+import annif.langsupport
+
 from . import analyzer
 
 
@@ -11,7 +13,10 @@ class SimplemmaAnalyzer(analyzer.Analyzer):
 
     def __init__(self, param: str, **kwargs) -> None:
         self.lang = param
+        self.lemmatizer = simplemma.Lemmatizer(
+            lemmatization_strategy=annif.langsupport.lemmatization_strategy
+        )
         super().__init__(**kwargs)
 
     def _normalize_word(self, word: str) -> str:
-        return simplemma.lemmatize(word, lang=self.lang)
+        return self.lemmatizer.lemmatize(word, lang=self.lang)

annif/langsupport.py

@@ -0,0 +1,9 @@
+"""Language support and language detection functionality for Annif"""
+
+from simplemma.strategies import DefaultStrategy
+from simplemma.strategies.dictionaries import DefaultDictionaryFactory
+
+LANG_CACHE_SIZE = 5  # How many language dictionaries to keep in memory at once (max)
+
+dictionary_factory = DefaultDictionaryFactory(cache_max_size=LANG_CACHE_SIZE)
+lemmatization_strategy = DefaultStrategy(dictionary_factory=dictionary_factory)

annif/transform/langfilter.py

@@ -4,9 +4,10 @@
 
 from typing import TYPE_CHECKING
 
-from simplemma.langdetect import in_target_language
+from simplemma import LanguageDetector
 
 import annif
+import annif.langsupport
 
 from . import transform
 
@@ -30,6 +31,10 @@ def __init__(
         self.text_min_length = int(text_min_length)
         self.sentence_min_length = int(sentence_min_length)
         self.min_ratio = float(min_ratio)
+        self.language_detector = LanguageDetector(
+            self.project.language,
+            lemmatization_strategy=annif.langsupport.lemmatization_strategy,
+        )
 
     def transform_fn(self, text: str) -> str:
         if len(text) < self.text_min_length:
@@ -40,7 +45,7 @@ def transform_fn(self, text: str) -> str:
             if len(sent) < self.sentence_min_length:
                 retained_sentences.append(sent)
                 continue
-            proportion = in_target_language(sent, lang=(self.project.language,))
+            proportion = self.language_detector.proportion_in_target_languages(sent)
             if proportion >= self.min_ratio:
                 retained_sentences.append(sent)
         return " ".join(retained_sentences)