theopenconversationkit#1606 Add Deepl translator module - first set o…

…f corrections
asus007 · Jul 9, 2024 · f8815e2 · f8815e2
1 parent ed3d5d2
commit f8815e2
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 63 deletions.
diff --git a/translator/deepl-translate/README.md b/translator/deepl-translate/README.md
@@ -0,0 +1,22 @@
+Module for translation with Deepl
+Here are the configurable variables:
+
+- tock_translator_deepl_target_languages : set of supported languages - ex : en,es
+- tock_translator_deepl_api_url : Deepl api url (default https://api.deepl.com/v2/translate)
+- tock_translator_deepl_api_key : Deepl api key to use (see your account)
+- tock_translator_deepl_glossaryId: glossary identifier to use in translation
+
+deepl decoumentation: https://developers.deepl.com/docs
+
+To integrate the module into a custom Tock Admin, pass the module as a parameter to the ai.tock.nlp.admin.startAdminServer() function.
+
+Example:
+
+package ai.tock.bot.admin
+
+import ai.tock.nlp.admin.startAdminServer
+import ai.tock.translator.deepl.deeplTranslatorModule
+
+fun main() {
+    startAdminServer(deeplTranslatorModule)
+}
diff --git a/translator/deepl-translate/pom.xml b/translator/deepl-translate/pom.xml
@@ -44,14 +44,8 @@
             <version>4.12.0</version>
         </dependency>
         <dependency>
-            <groupId>com.squareup.moshi</groupId>
-            <artifactId>moshi</artifactId>
-            <version>1.12.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.squareup.moshi</groupId>
-            <artifactId>moshi-kotlin</artifactId>
-            <version>1.12.0</version>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
         </dependency>
     </dependencies>
 

diff --git a/translator/deepl-translate/src/main/kotlin/DeeplClient.kt b/translator/deepl-translate/src/main/kotlin/DeeplClient.kt
@@ -15,29 +15,27 @@
  */
 
 package ai.tock.translator.deepl
-import okhttp3.MediaType.Companion.toMediaTypeOrNull
+
+import ai.tock.shared.jackson.mapper
+import com.fasterxml.jackson.module.kotlin.readValue
+import okhttp3.FormBody
 import okhttp3.OkHttpClient
 import okhttp3.Request
-import okhttp3.RequestBody.Companion.toRequestBody
-import com.squareup.moshi.Moshi
-import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory
 import java.io.IOException
 import java.util.regex.Pattern
 
-data class TranslationResponse(
+internal data class TranslationResponse(
     val translations: List<Translation>
 )
 
-data class Translation(
+internal data class Translation(
     val text: String
 )
 
 const val TAG_HANDLING = "xml"
 
-class DeeplClient(private val apiURL: String, private val apiKey: String) {
+internal class DeeplClient(private val apiURL: String, private val apiKey: String?) {
     private val client = OkHttpClient()
-    private val moshi = Moshi.Builder().add(KotlinJsonAdapterFactory()).build()
-    private val jsonAdapter = moshi.adapter(TranslationResponse::class.java)
 
     private fun replaceSpecificPlaceholders(text: String): Pair<String, List<String>> {
         // Store original placeholders for later restoration
@@ -63,35 +61,43 @@ class DeeplClient(private val apiURL: String, private val apiKey: String) {
         return resultText
     }
 
-    fun translate(text: String, sourceLang: String,targetLang: String,preserveFormatting: Boolean,glossaryId:String?): String? {
+    fun translate(
+        text: String,
+        sourceLang: String,
+        targetLang: String,
+        preserveFormatting: Boolean,
+        glossaryId: String?
+    ): String? {
         val (textWithPlaceholders, originalPlaceholders) = replaceSpecificPlaceholders(text)
 
-        val requestBody = buildString {
-            append("text=$textWithPlaceholders")
-            append("&source_lang=$sourceLang")
-            append("&target_lang=$targetLang")
-            append("&preserve_formatting=$preserveFormatting")
-            append("&tag_handling=$TAG_HANDLING")
+        val formBuilder = FormBody.Builder()
+
+        val requestBody = formBuilder
+            .add("text",textWithPlaceholders)
+            .add("source_lang",sourceLang)
+            .add("target_lang",targetLang)
+            .add("preserve_formatting", preserveFormatting.toString())
+            .add("tag_handling",TAG_HANDLING)
+            .build()
 
-            if (glossaryId != "default") {
-                append("&glossary=$glossaryId")
-            }
+        glossaryId?.let {
+            formBuilder.add("glossaryId", it)
         }
 
         val request = Request.Builder()
             .url(apiURL)
             .addHeader("Authorization", "DeepL-Auth-Key $apiKey")
-            .post(requestBody.trimIndent().toRequestBody("application/x-www-form-urlencoded".toMediaTypeOrNull()))
+            .post(requestBody)
             .build()
 
         client.newCall(request).execute().use { response ->
             if (!response.isSuccessful) throw IOException("Unexpected code $response")
 
             val responseBody = response.body?.string()
-            val translationResponse = jsonAdapter.fromJson(responseBody!!)
+            val translationResponse = mapper.readValue<TranslationResponse>(responseBody!!)
 
-            val translatedText = translationResponse?.translations?.firstOrNull()?.text
-            return translatedText?.let { revertSpecificPlaceholders(it,originalPlaceholders) }
+            val translatedText = translationResponse.translations.firstOrNull()?.text
+            return translatedText?.let { revertSpecificPlaceholders(it, originalPlaceholders) }
         }
     }
 }
diff --git a/translator/deepl-translate/src/main/kotlin/DeeplTranslatorEngine.kt b/translator/deepl-translate/src/main/kotlin/DeeplTranslatorEngine.kt
@@ -17,20 +17,26 @@
 package ai.tock.translator.deepl
 
 import ai.tock.shared.property
+import ai.tock.shared.propertyOrNull
 import ai.tock.translator.TranslatorEngine
 import org.apache.commons.text.StringEscapeUtils
 import java.util.Locale
 
 internal object DeeplTranslatorEngine : TranslatorEngine {
+    private val supportedLanguagesProperty = propertyOrNull("tock_translator_deepl_target_languages")
+    private val supportedLanguages: Set<String>? = supportedLanguagesProperty?.split(",")?.map { it.trim() }?.toSet()
 
-    private val deeplClient = DeeplClient(property ("tock_translator_deepl_api_url", "default"),property ("tock_translator_deepl_api_key", "default"))
-    private val glossaryId = property ("tock_translator_deepl_glossaryId", "default")
+    private val deeplClient = DeeplClient(
+        property("tock_translator_deepl_api_url", "https://api.deepl.com/v2/translate"),
+        propertyOrNull("tock_translator_deepl_api_key")
+    )
+    private val glossaryId = propertyOrNull("tock_translator_deepl_glossaryId")
     override val supportAdminTranslation: Boolean = true
 
     override fun translate(text: String, source: Locale, target: Locale): String {
         var translatedTextHTML4 = ""
         // Allows to filter translation on a specific language
-        if(target.language == property ("tock_translator_deepl_target_language", "en")) {
+        if (supportedLanguages?.contains(target.language) == true || supportedLanguages == null) {
             val translatedText = deeplClient.translate(text, source.language, target.language, true, glossaryId)
             translatedTextHTML4 = StringEscapeUtils.unescapeHtml4(translatedText)
         }

diff --git a/translator/deepl-translate/src/main/kotlin/DeeplTranslatorIoc.kt b/translator/deepl-translate/src/main/kotlin/DeeplTranslatorIoc.kt
@@ -16,22 +16,6 @@
 
 package ai.tock.translator.deepl
 
-/*
- * Copyright (C) 2017/2021 e-voyageurs technologies
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
 import ai.tock.translator.TranslatorEngine
 import com.github.salomonbrys.kodein.Kodein
 import com.github.salomonbrys.kodein.bind

diff --git a/translator/deepl-translate/src/test/kotlin/DeeplTranslateIntegrationTest.kt b/translator/deepl-translate/src/test/kotlin/DeeplTranslateIntegrationTest.kt
@@ -1,9 +1,3 @@
-import ai.tock.translator.deepl.DeeplTranslatorEngine
-import org.junit.jupiter.api.Disabled
-import org.junit.jupiter.api.Test
-import java.util.Locale
-import kotlin.test.assertEquals
-
 /*
  * Copyright (C) 2017/2021 e-voyageurs technologies
  *
@@ -20,6 +14,13 @@ import kotlin.test.assertEquals
  * limitations under the License.
  */
 
+package ai.tock.translator.deepl
+
+import org.junit.jupiter.api.Disabled
+import org.junit.jupiter.api.Test
+import java.util.Locale
+import kotlin.test.assertEquals
+
 /**
  * All these tests are disabled because it uses Deepl pro api that can be expensive
  */
@@ -38,16 +39,18 @@ class DeeplTranslateIntegrationTest {
     @Test
     @Disabled
     fun testWithEmoticonAndAntislash() {
-        val result = DeeplTranslatorEngine.translate("Bonjour, je suis l'Agent virtuel SNCF Voyageurs! \uD83E\uDD16\n" +
-                "Je vous informe sur l'état du trafic en temps réel.\n" +
-                "Dites-moi par exemple \"Mon train 6111 est-il à l'heure ?\", \"Aller à Saint-Lazare\", \"Prochains départs Gare de Lyon\" ...",
+        val result = DeeplTranslatorEngine.translate(
+            "Bonjour, je suis l'Agent virtuel SNCF Voyageurs! \uD83E\uDD16\n" +
+                    "Je vous informe sur l'état du trafic en temps réel.\n" +
+                    "Dites-moi par exemple \"Mon train 6111 est-il à l'heure ?\", \"Aller à Saint-Lazare\", \"Prochains départs Gare de Lyon\" ...",
             Locale.FRENCH,
             Locale.ENGLISH
         )
 
-        assertEquals("Hello, I'm the SNCF Voyageurs Virtual Agent! \uD83E\uDD16\n" +
-                "I inform you about traffic conditions in real time.\n" +
-                "Tell me for example \"Is my train 6111 on time?\", \"Going to Saint-Lazare\", \"Next departures Gare de Lyon\" ...",
+        assertEquals(
+            "Hello, I'm the SNCF Voyageurs Virtual Agent! \uD83E\uDD16\n" +
+                    "I inform you about traffic conditions in real time.\n" +
+                    "Tell me for example \"Is my train 6111 on time?\", \"Going to Saint-Lazare\", \"Next departures Gare de Lyon\" ...",
             result
         )
     }
@@ -60,7 +63,7 @@ class DeeplTranslateIntegrationTest {
             Locale.FRENCH,
             Locale.GERMAN
         )
-        assertEquals("Hallo, ich möchte nach {:city} {:date} reisen", result)
+        assertEquals("Hallo, ich würde gerne nach {:city} {:date} fahren.", result)
     }
 
     @Test