From f8815e27561721bf5c1732ffed26e962c10e9869 Mon Sep 17 00:00:00 2001 From: charles_moulhaud Date: Tue, 9 Jul 2024 14:32:01 +0200 Subject: [PATCH] #1606 Add Deepl translator module - first set of corrections --- translator/deepl-translate/README.md | 22 ++++++++ translator/deepl-translate/pom.xml | 10 +--- .../src/main/kotlin/DeeplClient.kt | 52 +++++++++++-------- .../src/main/kotlin/DeeplTranslatorEngine.kt | 12 +++-- .../src/main/kotlin/DeeplTranslatorIoc.kt | 16 ------ .../kotlin/DeeplTranslateIntegrationTest.kt | 29 ++++++----- 6 files changed, 78 insertions(+), 63 deletions(-) create mode 100644 translator/deepl-translate/README.md diff --git a/translator/deepl-translate/README.md b/translator/deepl-translate/README.md new file mode 100644 index 0000000000..76fe193ffe --- /dev/null +++ b/translator/deepl-translate/README.md @@ -0,0 +1,22 @@ +Module for translation with Deepl +Here are the configurable variables: + +- tock_translator_deepl_target_languages : set of supported languages - ex : en,es +- tock_translator_deepl_api_url : Deepl api url (default https://api.deepl.com/v2/translate) +- tock_translator_deepl_api_key : Deepl api key to use (see your account) +- tock_translator_deepl_glossaryId: glossary identifier to use in translation + +deepl decoumentation: https://developers.deepl.com/docs + +To integrate the module into a custom Tock Admin, pass the module as a parameter to the ai.tock.nlp.admin.startAdminServer() function. + +Example: + +package ai.tock.bot.admin + +import ai.tock.nlp.admin.startAdminServer +import ai.tock.translator.deepl.deeplTranslatorModule + +fun main() { + startAdminServer(deeplTranslatorModule) +} \ No newline at end of file diff --git a/translator/deepl-translate/pom.xml b/translator/deepl-translate/pom.xml index 2701a6608a..887c6fa7ec 100644 --- a/translator/deepl-translate/pom.xml +++ b/translator/deepl-translate/pom.xml @@ -44,14 +44,8 @@ 4.12.0 - com.squareup.moshi - moshi - 1.12.0 - - - com.squareup.moshi - moshi-kotlin - 1.12.0 + com.fasterxml.jackson.core + jackson-core diff --git a/translator/deepl-translate/src/main/kotlin/DeeplClient.kt b/translator/deepl-translate/src/main/kotlin/DeeplClient.kt index fbeb4909e3..2a4ab377c7 100644 --- a/translator/deepl-translate/src/main/kotlin/DeeplClient.kt +++ b/translator/deepl-translate/src/main/kotlin/DeeplClient.kt @@ -15,29 +15,27 @@ */ package ai.tock.translator.deepl -import okhttp3.MediaType.Companion.toMediaTypeOrNull + +import ai.tock.shared.jackson.mapper +import com.fasterxml.jackson.module.kotlin.readValue +import okhttp3.FormBody import okhttp3.OkHttpClient import okhttp3.Request -import okhttp3.RequestBody.Companion.toRequestBody -import com.squareup.moshi.Moshi -import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory import java.io.IOException import java.util.regex.Pattern -data class TranslationResponse( +internal data class TranslationResponse( val translations: List ) -data class Translation( +internal data class Translation( val text: String ) const val TAG_HANDLING = "xml" -class DeeplClient(private val apiURL: String, private val apiKey: String) { +internal class DeeplClient(private val apiURL: String, private val apiKey: String?) { private val client = OkHttpClient() - private val moshi = Moshi.Builder().add(KotlinJsonAdapterFactory()).build() - private val jsonAdapter = moshi.adapter(TranslationResponse::class.java) private fun replaceSpecificPlaceholders(text: String): Pair> { // Store original placeholders for later restoration @@ -63,35 +61,43 @@ class DeeplClient(private val apiURL: String, private val apiKey: String) { return resultText } - fun translate(text: String, sourceLang: String,targetLang: String,preserveFormatting: Boolean,glossaryId:String?): String? { + fun translate( + text: String, + sourceLang: String, + targetLang: String, + preserveFormatting: Boolean, + glossaryId: String? + ): String? { val (textWithPlaceholders, originalPlaceholders) = replaceSpecificPlaceholders(text) - val requestBody = buildString { - append("text=$textWithPlaceholders") - append("&source_lang=$sourceLang") - append("&target_lang=$targetLang") - append("&preserve_formatting=$preserveFormatting") - append("&tag_handling=$TAG_HANDLING") + val formBuilder = FormBody.Builder() + + val requestBody = formBuilder + .add("text",textWithPlaceholders) + .add("source_lang",sourceLang) + .add("target_lang",targetLang) + .add("preserve_formatting", preserveFormatting.toString()) + .add("tag_handling",TAG_HANDLING) + .build() - if (glossaryId != "default") { - append("&glossary=$glossaryId") - } + glossaryId?.let { + formBuilder.add("glossaryId", it) } val request = Request.Builder() .url(apiURL) .addHeader("Authorization", "DeepL-Auth-Key $apiKey") - .post(requestBody.trimIndent().toRequestBody("application/x-www-form-urlencoded".toMediaTypeOrNull())) + .post(requestBody) .build() client.newCall(request).execute().use { response -> if (!response.isSuccessful) throw IOException("Unexpected code $response") val responseBody = response.body?.string() - val translationResponse = jsonAdapter.fromJson(responseBody!!) + val translationResponse = mapper.readValue(responseBody!!) - val translatedText = translationResponse?.translations?.firstOrNull()?.text - return translatedText?.let { revertSpecificPlaceholders(it,originalPlaceholders) } + val translatedText = translationResponse.translations.firstOrNull()?.text + return translatedText?.let { revertSpecificPlaceholders(it, originalPlaceholders) } } } } \ No newline at end of file diff --git a/translator/deepl-translate/src/main/kotlin/DeeplTranslatorEngine.kt b/translator/deepl-translate/src/main/kotlin/DeeplTranslatorEngine.kt index debbdf9246..88c14971cd 100644 --- a/translator/deepl-translate/src/main/kotlin/DeeplTranslatorEngine.kt +++ b/translator/deepl-translate/src/main/kotlin/DeeplTranslatorEngine.kt @@ -17,20 +17,26 @@ package ai.tock.translator.deepl import ai.tock.shared.property +import ai.tock.shared.propertyOrNull import ai.tock.translator.TranslatorEngine import org.apache.commons.text.StringEscapeUtils import java.util.Locale internal object DeeplTranslatorEngine : TranslatorEngine { + private val supportedLanguagesProperty = propertyOrNull("tock_translator_deepl_target_languages") + private val supportedLanguages: Set? = supportedLanguagesProperty?.split(",")?.map { it.trim() }?.toSet() - private val deeplClient = DeeplClient(property ("tock_translator_deepl_api_url", "default"),property ("tock_translator_deepl_api_key", "default")) - private val glossaryId = property ("tock_translator_deepl_glossaryId", "default") + private val deeplClient = DeeplClient( + property("tock_translator_deepl_api_url", "https://api.deepl.com/v2/translate"), + propertyOrNull("tock_translator_deepl_api_key") + ) + private val glossaryId = propertyOrNull("tock_translator_deepl_glossaryId") override val supportAdminTranslation: Boolean = true override fun translate(text: String, source: Locale, target: Locale): String { var translatedTextHTML4 = "" // Allows to filter translation on a specific language - if(target.language == property ("tock_translator_deepl_target_language", "en")) { + if (supportedLanguages?.contains(target.language) == true || supportedLanguages == null) { val translatedText = deeplClient.translate(text, source.language, target.language, true, glossaryId) translatedTextHTML4 = StringEscapeUtils.unescapeHtml4(translatedText) } diff --git a/translator/deepl-translate/src/main/kotlin/DeeplTranslatorIoc.kt b/translator/deepl-translate/src/main/kotlin/DeeplTranslatorIoc.kt index c2954fc393..0c2497defa 100644 --- a/translator/deepl-translate/src/main/kotlin/DeeplTranslatorIoc.kt +++ b/translator/deepl-translate/src/main/kotlin/DeeplTranslatorIoc.kt @@ -16,22 +16,6 @@ package ai.tock.translator.deepl -/* - * Copyright (C) 2017/2021 e-voyageurs technologies - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - import ai.tock.translator.TranslatorEngine import com.github.salomonbrys.kodein.Kodein import com.github.salomonbrys.kodein.bind diff --git a/translator/deepl-translate/src/test/kotlin/DeeplTranslateIntegrationTest.kt b/translator/deepl-translate/src/test/kotlin/DeeplTranslateIntegrationTest.kt index 7c90640ab1..a8775a8617 100644 --- a/translator/deepl-translate/src/test/kotlin/DeeplTranslateIntegrationTest.kt +++ b/translator/deepl-translate/src/test/kotlin/DeeplTranslateIntegrationTest.kt @@ -1,9 +1,3 @@ -import ai.tock.translator.deepl.DeeplTranslatorEngine -import org.junit.jupiter.api.Disabled -import org.junit.jupiter.api.Test -import java.util.Locale -import kotlin.test.assertEquals - /* * Copyright (C) 2017/2021 e-voyageurs technologies * @@ -20,6 +14,13 @@ import kotlin.test.assertEquals * limitations under the License. */ +package ai.tock.translator.deepl + +import org.junit.jupiter.api.Disabled +import org.junit.jupiter.api.Test +import java.util.Locale +import kotlin.test.assertEquals + /** * All these tests are disabled because it uses Deepl pro api that can be expensive */ @@ -38,16 +39,18 @@ class DeeplTranslateIntegrationTest { @Test @Disabled fun testWithEmoticonAndAntislash() { - val result = DeeplTranslatorEngine.translate("Bonjour, je suis l'Agent virtuel SNCF Voyageurs! \uD83E\uDD16\n" + - "Je vous informe sur l'état du trafic en temps réel.\n" + - "Dites-moi par exemple \"Mon train 6111 est-il à l'heure ?\", \"Aller à Saint-Lazare\", \"Prochains départs Gare de Lyon\" ...", + val result = DeeplTranslatorEngine.translate( + "Bonjour, je suis l'Agent virtuel SNCF Voyageurs! \uD83E\uDD16\n" + + "Je vous informe sur l'état du trafic en temps réel.\n" + + "Dites-moi par exemple \"Mon train 6111 est-il à l'heure ?\", \"Aller à Saint-Lazare\", \"Prochains départs Gare de Lyon\" ...", Locale.FRENCH, Locale.ENGLISH ) - assertEquals("Hello, I'm the SNCF Voyageurs Virtual Agent! \uD83E\uDD16\n" + - "I inform you about traffic conditions in real time.\n" + - "Tell me for example \"Is my train 6111 on time?\", \"Going to Saint-Lazare\", \"Next departures Gare de Lyon\" ...", + assertEquals( + "Hello, I'm the SNCF Voyageurs Virtual Agent! \uD83E\uDD16\n" + + "I inform you about traffic conditions in real time.\n" + + "Tell me for example \"Is my train 6111 on time?\", \"Going to Saint-Lazare\", \"Next departures Gare de Lyon\" ...", result ) } @@ -60,7 +63,7 @@ class DeeplTranslateIntegrationTest { Locale.FRENCH, Locale.GERMAN ) - assertEquals("Hallo, ich möchte nach {:city} {:date} reisen", result) + assertEquals("Hallo, ich würde gerne nach {:city} {:date} fahren.", result) } @Test