From 9fbd9446ee1916774cc5967bee443f1c2bf348fb Mon Sep 17 00:00:00 2001 From: Mark Allen <3417310+maallen@users.noreply.github.com> Date: Tue, 5 Nov 2024 09:13:14 +0000 Subject: [PATCH] Added PUSH_AI_TRANSLATION to third party sync (#166) * Batch delete in loop iteration * ThirdPartySync updates compiling * AI translation import to Smartling via POST_TRANSLATION upload * Added push ai translations support to json sync * Added unit tests for third party ai translation push * Remove dead code * Code review updates * Update variant status name * Documentation and renamed variable --- .../mojito/rest/ThirdPartySyncAction.java | 3 +- .../l10n/mojito/entity/TMTextUnitVariant.java | 7 +- .../rest/thirdparty/ThirdPartySyncAction.java | 3 +- .../ai/translation/AITranslateCronJob.java | 6 +- .../ai/translation/AITranslationService.java | 66 ++- .../service/thirdparty/ThirdPartyService.java | 34 ++ .../service/thirdparty/ThirdPartyTMS.java | 10 + .../thirdparty/ThirdPartyTMSInMemory.java | 11 + .../thirdparty/ThirdPartyTMSSmartling.java | 293 +++++++++++- .../ThirdPartyTMSSmartlingWithJson.java | 179 +++++-- .../smartling/SmartlingResultProcessor.java | 4 + .../service/tm/search/StatusFilter.java | 2 + .../mojito/service/tm/search/TextUnitDTO.java | 9 + .../search/TextUnitDTONativeObjectMapper.java | 4 + .../service/tm/search/TextUnitSearcher.java | 26 +- .../tm/search/TextUnitSearcherParameters.java | 16 + .../mojito/smartling/SmartlingClient.java | 22 +- .../translation/AITranslateCronJobTest.java | 7 +- .../StubSmartlingResultProcessor.java | 8 + .../ThirdPartyTMSSmartlingTest.java | 435 +++++++++++++++++- .../ThirdPartyTMSSmartlingWithJsonTest.java | 12 +- 21 files changed, 1049 insertions(+), 108 deletions(-) diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/ThirdPartySyncAction.java b/restclient/src/main/java/com/box/l10n/mojito/rest/ThirdPartySyncAction.java index 76fce0802e..d542080012 100644 --- a/restclient/src/main/java/com/box/l10n/mojito/rest/ThirdPartySyncAction.java +++ b/restclient/src/main/java/com/box/l10n/mojito/rest/ThirdPartySyncAction.java @@ -6,5 +6,6 @@ public enum ThirdPartySyncAction { PULL, PULL_SOURCE, MAP_TEXTUNIT, - PUSH_SCREENSHOT + PUSH_SCREENSHOT, + PUSH_AI_TRANSLATION, } diff --git a/webapp/src/main/java/com/box/l10n/mojito/entity/TMTextUnitVariant.java b/webapp/src/main/java/com/box/l10n/mojito/entity/TMTextUnitVariant.java index 99808224ae..5c1a455fcb 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/entity/TMTextUnitVariant.java +++ b/webapp/src/main/java/com/box/l10n/mojito/entity/TMTextUnitVariant.java @@ -69,9 +69,14 @@ public enum Status { */ REVIEW_NEEDED, + /** Indicates that the text unit has been machine translated in Mojito automatically. */ MT_TRANSLATED, - MT_REVIEW, + /** + * Indicates that the text unit has been machine translated in Mojito automatically and has been + * sent for third party review. + */ + MT_REVIEW_NEEDED, /** A string that doesn't need any work to be performed on it. */ APPROVED, /** It was overridden in Mojito, so it won't be updated during third-party sync */ diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/thirdparty/ThirdPartySyncAction.java b/webapp/src/main/java/com/box/l10n/mojito/rest/thirdparty/ThirdPartySyncAction.java index e46aedd2ed..4072e7f310 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/rest/thirdparty/ThirdPartySyncAction.java +++ b/webapp/src/main/java/com/box/l10n/mojito/rest/thirdparty/ThirdPartySyncAction.java @@ -6,5 +6,6 @@ public enum ThirdPartySyncAction { PULL, PULL_SOURCE, MAP_TEXTUNIT, - PUSH_SCREENSHOT + PUSH_SCREENSHOT, + PUSH_AI_TRANSLATION } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/ai/translation/AITranslateCronJob.java b/webapp/src/main/java/com/box/l10n/mojito/service/ai/translation/AITranslateCronJob.java index 5fcef410f9..d7abbb9d45 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/ai/translation/AITranslateCronJob.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/ai/translation/AITranslateCronJob.java @@ -21,8 +21,10 @@ import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -271,6 +273,7 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution pendingMTs = tmTextUnitPendingMTRepository.findBatch(aiTranslationConfiguration.getBatchSize()); logger.info("Processing {} pending MTs", pendingMTs.size()); + Queue textUnitsToClearPendingMT = new ConcurrentLinkedQueue<>(); List> futures = pendingMTs.stream() @@ -295,7 +298,7 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution logger.debug( "Sending pending MT for tmTextUnitId: {} for deletion", pendingMT.getTmTextUnitId()); - aiTranslationService.sendForDeletion(pendingMT); + textUnitsToClearPendingMT.add(pendingMT); } } }, @@ -304,6 +307,7 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution // Wait for all tasks in this batch to complete CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + aiTranslationService.deleteBatch(textUnitsToClearPendingMT); } while (!pendingMTs.isEmpty()); } finally { shutdownExecutor(executorService); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/ai/translation/AITranslationService.java b/webapp/src/main/java/com/box/l10n/mojito/service/ai/translation/AITranslationService.java index e509861be2..6dcf7b501c 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/ai/translation/AITranslationService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/ai/translation/AITranslationService.java @@ -1,5 +1,7 @@ package com.box.l10n.mojito.service.ai.translation; +import static com.box.l10n.mojito.entity.TMTextUnitVariant.Status.MT_REVIEW_NEEDED; + import com.box.l10n.mojito.JSR310Migration; import com.box.l10n.mojito.entity.PromptType; import com.box.l10n.mojito.entity.TmTextUnitPendingMT; @@ -7,13 +9,14 @@ import com.box.l10n.mojito.service.repository.RepositoryRepository; import com.box.l10n.mojito.service.tm.TMTextUnitVariantRepository; import com.google.common.collect.Lists; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; import jakarta.transaction.Transactional; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.sql.Timestamp; import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.Set; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -21,10 +24,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Component; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Sinks; @Component @ConditionalOnProperty(value = "l10n.ai.translation.enabled", havingValue = "true") @@ -51,9 +53,6 @@ public class AITranslationService { @Value("${l10n.ai.translation.maxTextUnitsAIRequest:1000}") int maxTextUnitsAIRequest; - private final Sinks.Many pendingMTDeletionSink = - Sinks.many().multicast().onBackpressureBuffer(); - @Transactional public void createPendingMTEntitiesInBatches(Long repositoryId, Set tmTextUnitIds) { if (tmTextUnitIds.size() > maxTextUnitsAIRequest) { @@ -72,9 +71,40 @@ public void createPendingMTEntitiesInBatches(Long repositoryId, Set tmText } } - protected void sendForDeletion(TmTextUnitPendingMT pendingMT) { - logger.debug("Sending pending MT for deletion: {}", pendingMT); - pendingMTDeletionSink.tryEmitNext(pendingMT); + @Transactional + public void updateVariantStatusToMTReview(List currentVariantIds) { + + for (int i = 0; i < currentVariantIds.size(); i += batchSize) { + logger.debug("Updating variant statuses to MT_REVIEW in batches of {}", batchSize); + int end = Math.min(i + batchSize, currentVariantIds.size()); + List updateBatch = currentVariantIds.subList(i, end); + executeVariantStatusUpdatesToMTReview(updateBatch); + } + } + + private void executeVariantStatusUpdatesToMTReview(List updateBatch) { + String sql = + "UPDATE tm_text_unit_variant " + + "SET status = ? " + + "WHERE id IN (" + + "SELECT tucv.tm_text_unit_variant_id " + + "FROM tm_text_unit_current_variant tucv " + + "WHERE tucv.id = ?)"; + + jdbcTemplate.batchUpdate( + sql, + new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + ps.setString(1, MT_REVIEW_NEEDED.name()); + ps.setLong(2, updateBatch.get(i)); + } + + @Override + public int getBatchSize() { + return updateBatch.size(); + } + }); } private void createPendingMTEntitiesInBatches(Set tmTextUnitIds) { @@ -177,7 +207,7 @@ private void insertMultiRowTextUnitCurrentVariants( } @Transactional - private void deleteBatch(List batch) { + protected void deleteBatch(Queue batch) { if (batch.isEmpty()) { logger.debug("No pending MTs to delete"); return; @@ -201,18 +231,4 @@ private static TmTextUnitPendingMT createTmTextUnitPendingMT(Long tmTextUnitId) tmTextUnitPendingMT.setCreatedDate(JSR310Migration.newDateTimeEmptyCtor()); return tmTextUnitPendingMT; } - - @PostConstruct - public void init() { - Flux flux = pendingMTDeletionSink.asFlux(); - - flux.bufferTimeout(batchSize, timeout) - .filter(batch -> !batch.isEmpty()) - .subscribe(this::deleteBatch); - } - - @PreDestroy - public void destroy() { - pendingMTDeletionSink.tryEmitComplete(); - } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java index cecd5686a3..b37cc6ce35 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java @@ -180,6 +180,18 @@ void syncMojitoWithThirdPartyTMS( if (actions.contains(ThirdPartySyncAction.PUSH_SCREENSHOT)) { uploadScreenshotsAndCreateMappings(repository, thirdPartyProjectId, currentTask); } + if (actions.contains(ThirdPartySyncAction.PUSH_AI_TRANSLATION)) { + pushAITranslations( + thirdPartyProjectId, + pluralSeparator, + localeMapping, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + includeTextUnitsWithPattern, + options, + repository, + currentTask); + } } @Pollable(message = "Push source strings to third party service.") @@ -290,6 +302,28 @@ void mapMojitoAndThirdPartyTextUnits( e -> mapThirdPartyTextUnitsToTextUnitDTOs(e.getKey(), e.getValue(), pluralSeparator)); } + @Pollable(message = "Push AI translations to third party service.") + void pushAITranslations( + String thirdPartyProjectId, + String pluralSeparator, + String localeMapping, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitsWithPattern, + List options, + Repository repository, + @ParentTask PollableTask currentTask) { + thirdPartyTMS.pushAITranslations( + repository, + thirdPartyProjectId, + pluralSeparator, + parseLocaleMapping(localeMapping), + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + includeTextUnitsWithPattern, + options); + } + void mapThirdPartyTextUnitsToTextUnitDTOs( Asset asset, List thirdPartyTextUnitsToMap, String pluralSeparator) { logger.debug("Map third party text units to text unit DTOs for asset: {}", asset.getId()); diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMS.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMS.java index 87ae24bc7b..fb7e1c33ab 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMS.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMS.java @@ -96,4 +96,14 @@ void pullSource( String projectId, List optionList, Map localeMapping); + + void pushAITranslations( + Repository repository, + String projectId, + String pluralSeparator, + Map localeMapping, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitsWithPattern, + List optionList); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSInMemory.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSInMemory.java index 30117be7bd..24edf3fb56 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSInMemory.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSInMemory.java @@ -117,4 +117,15 @@ public void pullSource( String projectId, List optionList, Map localeMapping) {} + + @Override + public void pushAITranslations( + Repository repository, + String projectId, + String pluralSeparator, + Map localeMapping, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitsWithPattern, + List optionList) {} } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java index 7f436a878e..c02b6e9643 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java @@ -13,6 +13,7 @@ import com.box.l10n.mojito.quartz.QuartzJobInfo; import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler; import com.box.l10n.mojito.service.ai.translation.AITranslationConfiguration; +import com.box.l10n.mojito.service.ai.translation.AITranslationService; import com.box.l10n.mojito.service.assetExtraction.AssetTextUnitToTMTextUnitRepository; import com.box.l10n.mojito.service.pollableTask.PollableFuture; import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFile; @@ -95,6 +96,7 @@ public class ThirdPartyTMSSmartling implements ThirdPartyTMS { private final ThirdPartyTMSSmartlingWithJson thirdPartyTMSSmartlingWithJson; private final ThirdPartyTMSSmartlingGlossary thirdPartyTMSSmartlingGlossary; private final AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository; + private final AITranslationService aiTranslationService; private final MeterRegistry meterRegistry; @@ -109,6 +111,9 @@ protected static String getSmartlingLocale(Map localeMapping, St return localeMapping.getOrDefault(localeTag, localeTag); } + private final String PUBLISHED = "PUBLISHED"; + private final String POST_TRANSLATION = "POST_TRANSLATION"; + @Autowired public ThirdPartyTMSSmartling( SmartlingClient smartlingClient, @@ -121,7 +126,8 @@ public ThirdPartyTMSSmartling( AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository, MeterRegistry meterRegistry, QuartzPollableTaskScheduler quartzPollableTaskScheduler, - AITranslationConfiguration aiTranslationConfiguration) { + AITranslationConfiguration aiTranslationConfiguration, + AITranslationService aiTranslationService) { this( smartlingClient, textUnitSearcher, @@ -134,7 +140,8 @@ public ThirdPartyTMSSmartling( DEFAULT_BATCH_SIZE, meterRegistry, quartzPollableTaskScheduler, - aiTranslationConfiguration); + aiTranslationConfiguration, + aiTranslationService); } public ThirdPartyTMSSmartling( @@ -149,7 +156,8 @@ public ThirdPartyTMSSmartling( int batchSize, MeterRegistry meterRegistry, QuartzPollableTaskScheduler quartzPollableTaskScheduler, - AITranslationConfiguration aiTranslationConfiguration) { + AITranslationConfiguration aiTranslationConfiguration, + AITranslationService aiTranslationService) { this.smartlingClient = smartlingClient; this.assetPathAndTextUnitNameKeys = assetPathAndTextUnitNameKeys; this.textUnitBatchImporterService = textUnitBatchImporterService; @@ -162,6 +170,7 @@ public ThirdPartyTMSSmartling( this.meterRegistry = meterRegistry; this.quartzPollableTaskScheduler = quartzPollableTaskScheduler; this.aiTranslationConfiguration = aiTranslationConfiguration; + this.aiTranslationService = aiTranslationService; } @Override @@ -673,7 +682,8 @@ public void pushTranslations( options, localeMapping, Prefix.SINGULAR, - filterTmTextUnitIds)), + filterTmTextUnitIds, + PUBLISHED)), mapWithIndex( partitionPlurals( repository.getId(), @@ -693,7 +703,8 @@ public void pushTranslations( options, localeMapping, Prefix.PLURAL, - filterTmTextUnitIds)))) + filterTmTextUnitIds, + PUBLISHED)))) .collect(Collectors.toList()); resultProcessor.processPushTranslations(result, options); @@ -733,6 +744,140 @@ public void pullSource( } } + @Override + public void pushAITranslations( + Repository repository, + String projectId, + String pluralSeparator, + Map localeMapping, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitsWithPattern, + List optionList) { + try (var timer = + Timer.resource(meterRegistry, "SmartlingSync.pushAITranslations") + .tag("repository", repository.getName())) { + SmartlingOptions options = SmartlingOptions.parseList(optionList); + if (options.isJsonSync()) { + thirdPartyTMSSmartlingWithJson.pushAiTranslations( + repository, + projectId, + localeMapping, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + includeTextUnitsWithPattern); + } + + AndroidStringDocumentMapper mapper = new AndroidStringDocumentMapper(pluralSeparator, null); + + Set filterTmTextUnitIds = getFilterTmTextUnitIdsForPushTranslation(options); + + List result = + repository.getRepositoryLocales().stream() + .map(l -> l.getLocale().getBcp47Tag()) + .filter( + localeTag -> + !localeTag.equalsIgnoreCase(repository.getSourceLocale().getBcp47Tag())) + .flatMap( + localeTag -> { + Map> singularsByUploadedFileUri = + partitionSingulars( + repository.getId(), + localeTag, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + includeTextUnitsWithPattern, + StatusFilter.MT_TRANSLATED, + true) + .flatMap(List::stream) + .collect(Collectors.groupingBy(TextUnitDTO::getUploadedFileUri)); + + Map> pluralsByUploadedFileUri = + partitionPlurals( + repository.getId(), + localeTag, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + options.getPluralFixForLocales(), + includeTextUnitsWithPattern, + StatusFilter.MT_TRANSLATED, + true) + .flatMap(List::stream) + .collect(Collectors.groupingBy(TextUnitDTO::getUploadedFileUri)); + + Stream singularFiles = + singularsByUploadedFileUri.entrySet().stream() + .map( + entry -> + uploadAiTranslationBatch( + entry.getValue(), + entry.getKey(), + localeTag, + mapper, + repository, + projectId, + options, + localeMapping, + filterTmTextUnitIds)); + + Stream pluralFiles = + pluralsByUploadedFileUri.entrySet().stream() + .map( + entry -> + uploadAiTranslationBatch( + entry.getValue(), + entry.getKey(), + localeTag, + mapper, + repository, + projectId, + options, + localeMapping, + filterTmTextUnitIds)); + + return Stream.concat(singularFiles, pluralFiles); + }) + .collect(Collectors.toList()); + + resultProcessor.processPushAiTranslations(result, options); + } + } + + private SmartlingFile uploadAiTranslationBatch( + List batch, + String uploadedFileUri, + String localeTag, + AndroidStringDocumentMapper mapper, + Repository repository, + String projectId, + SmartlingOptions options, + Map localeMapping, + Set filterTmTextUnitIds) { + + SmartlingFile file = + uploadAiTranslationFile( + batch, + localeTag, + uploadedFileUri, + mapper, + repository, + projectId, + options, + localeMapping, + filterTmTextUnitIds); + aiTranslationService.updateVariantStatusToMTReview( + batch.stream().map(TextUnitDTO::getTmTextUnitCurrentVariantId).toList()); + meterRegistry + .counter( + "SmartlingSync.uploadAiTranslationsBatch", + "repository", + repository.getName(), + "jsonSync", + "false") + .increment(batch.size()); + return file; + } + private SmartlingFile processTranslationBatch( List batch, Long batchNumber, @@ -743,7 +888,8 @@ private SmartlingFile processTranslationBatch( SmartlingOptions options, Map localeMapping, Prefix filePrefix, - Set filterTmTextUnitIds) { + Set filterTmTextUnitIds, + String translationState) { try (var timer = Timer.resource(meterRegistry, "SmartlingSync.processTranslationBatch") @@ -793,7 +939,8 @@ private SmartlingFile processTranslationBatch( getSmartlingLocale(localeMapping, localeTag), file.getFileContent(), options.getPlaceholderFormat(), - options.getCustomPlaceholderFormat())) + options.getCustomPlaceholderFormat(), + translationState)) .retryWhen( smartlingClient .getRetryConfiguration() @@ -820,6 +967,87 @@ private SmartlingFile processTranslationBatch( } } + private SmartlingFile uploadAiTranslationFile( + List batch, + String localeTag, + String fileName, + AndroidStringDocumentMapper mapper, + Repository repository, + String projectId, + SmartlingOptions options, + Map localeMapping, + Set filterTmTextUnitIds) { + + try (var timer = + Timer.resource(meterRegistry, "SmartlingSync.processTranslationBatch") + .tag("repository", repository.getName()) + .tag("locale", localeTag)) { + + logger.debug("Process translation batch for file: {}", fileName); + List fileBatch = batch; + SmartlingFile file = new SmartlingFile(); + file.setFileName(fileName + "_" + localeTag); + + try { + logger.debug("Save target file to: {}", file.getFileName()); + + if (filterTmTextUnitIds != null) { + fileBatch = + fileBatch.stream() + .filter( + textUnitDTO -> filterTmTextUnitIds.contains(textUnitDTO.getTmTextUnitId())) + .collect(Collectors.toList()); + } + + AndroidStringDocumentWriter writer = + new AndroidStringDocumentWriter(mapper.readFromTargetTextUnits(fileBatch)); + file.setFileContent(writer.toText()); + + } catch (ParserConfigurationException | TransformerException e) { + logger.error("An error occurred when processing a translation batch", e); + throw new RuntimeException(e); + } + + if (!options.isDryRun()) { + logger.debug( + "Pushing Android file to Smartling project: {} and locale: {}", projectId, localeTag); + Mono.fromCallable( + () -> + smartlingClient.uploadLocalizedFile( + projectId, + fileName, + "android", + getSmartlingLocale(localeMapping, localeTag), + file.getFileContent(), + options.getPlaceholderFormat(), + options.getCustomPlaceholderFormat(), + POST_TRANSLATION)) + .retryWhen( + smartlingClient + .getRetryConfiguration() + .doBeforeRetry( + e -> + logger.info( + String.format( + "Retrying after failure to upload localized file: %s, project id: %s", + fileName, projectId), + e.failure()))) + .doOnError( + e -> { + String msg = + String.format( + "Error uploading localized file to Smartling for file %s in project %s", + fileName, projectId); + logger.error(msg, e); + throw new SmartlingClientException(msg, e); + }) + .block(); + } + + return file; + } + } + private Stream> partitionSingulars( Long repositoryId, String localeTag, @@ -858,6 +1086,31 @@ private Stream> partitionSingulars( return partitionedStream(parameters, textUnitSearcher::search); } + private Stream> partitionSingulars( + Long repositoryId, + String localeTag, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitWithPattern, + StatusFilter statusFilter, + boolean isRetrieveFileUploadUri) { + TextUnitSearcherParameters parameters = + this.baseParams() + .repositoryId(repositoryId) + .localeTags(ImmutableList.of(localeTag)) + .skipTextUnitWithPattern(skipTextUnitsWithPattern) + .skipAssetPathWithPattern(skipAssetsWithPathPattern) + .pluralFormsFiltered(true) + .pluralFormsExcluded(true) + .includeTextUnitsWithPattern(includeTextUnitWithPattern) + .statusFilter(statusFilter) + .isOrderedByTextUnitID(true) + .shouldRetrieveUploadedFileUri(isRetrieveFileUploadUri) + .build(); + + return partitionedStream(parameters, textUnitSearcher::search); + } + private Stream> partitionPlurals( Long repositoryId, String localeTag, @@ -884,11 +1137,30 @@ private Stream> partitionPlurals( String skipAssetsWithPathPattern, Set pluralFixForLocales, String includeTextUnitsWithPattern) { + return partitionPlurals( + repositoryId, + localeTag, + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + pluralFixForLocales, + includeTextUnitsWithPattern, + null, + false); + } + private Stream> partitionPlurals( + Long repositoryId, + String localeTag, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + Set pluralFixForLocales, + String includeTextUnitsWithPattern, + StatusFilter statusFilter, + boolean isRetrieveFileUploadUri) { Function> searchFunction = textUnitSearcher::search; - if (pluralFixForLocales.contains(localeTag)) { + if (pluralFixForLocales != null && pluralFixForLocales.contains(localeTag)) { searchFunction = searchFunction.andThen( textUnits -> @@ -908,8 +1180,13 @@ private Stream> partitionPlurals( .pluralFormOther("%") .includeTextUnitsWithPattern(includeTextUnitsWithPattern) .isOrderedByTextUnitID(true) + .shouldRetrieveUploadedFileUri(isRetrieveFileUploadUri) .build(); + if (statusFilter != null) { + parameters.setStatusFilter(statusFilter); + } + return partitionedStream(parameters, searchFunction); } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java index 4343368218..79228bef3d 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJson.java @@ -7,6 +7,7 @@ import com.box.l10n.mojito.entity.RepositoryLocale; import com.box.l10n.mojito.iterators.PageFetcherOffsetAndLimitSplitIterator; import com.box.l10n.mojito.iterators.Spliterators; +import com.box.l10n.mojito.service.ai.translation.AITranslationConfiguration; import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingJsonConverter; import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingOptions; import com.box.l10n.mojito.service.tm.importer.TextUnitBatchImporterService; @@ -32,6 +33,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.stream.StreamSupport; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -61,6 +63,8 @@ public class ThirdPartyTMSSmartlingWithJson { ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository; + AITranslationConfiguration aiTranslationConfiguration; + int batchSize = 5000; public ThirdPartyTMSSmartlingWithJson( @@ -70,7 +74,8 @@ public ThirdPartyTMSSmartlingWithJson( TextUnitBatchImporterService textUnitBatchImporterService, SmartlingJsonKeys smartlingJsonKeys, MeterRegistry meterRegistry, - ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository) { + ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepository, + AITranslationConfiguration aiTranslationConfiguration) { this.smartlingClient = smartlingClient; this.smartlingJsonConverter = smartlingJsonConverter; this.textUnitSearcher = textUnitSearcher; @@ -78,6 +83,7 @@ public ThirdPartyTMSSmartlingWithJson( this.smartlingJsonKeys = smartlingJsonKeys; this.meterRegistry = meterRegistry; this.thirdPartyFileChecksumRepository = thirdPartyFileChecksumRepository; + this.aiTranslationConfiguration = aiTranslationConfiguration; } void push( @@ -255,6 +261,101 @@ && isFileEqualToPreviousRun( }); } + public void pushAiTranslations( + Repository repository, + String projectId, + Map localeMapping, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitsWithPattern) { + + logger.info( + "Push AI translations from repository: {} into project: {}", + repository.getName(), + projectId); + + try (var timer = + Timer.resource(meterRegistry, "ThirdPartyTMSSmartlingWithJson.pushAiTranslations") + .tag("repository", repository.getName())) { + getRepositoryLocaleWithoutRootStream(repository) + .forEach( + repositoryLocale -> { + Map> textUnitsByUploadedFileUri = + StreamSupport.stream( + getTargetTextUnitIterator( + repository, + repositoryLocale.getLocale().getId(), + skipTextUnitsWithPattern, + skipAssetsWithPathPattern, + includeTextUnitsWithPattern, + StatusFilter.MT_TRANSLATED), + false) + .collect(Collectors.groupingBy(TextUnitDTO::getUploadedFileUri)); + + textUnitsByUploadedFileUri.forEach( + (uploadedFileUri, textUnitDTOS) -> { + try (var timer2 = + Timer.resource( + meterRegistry, + "ThirdPartyTMSSmartlingWithJson.pushAiTranslations.batch") + .tag("repository", repository.getName())) { + + uploadLocalizedFile( + projectId, + localeMapping, + repositoryLocale, + uploadedFileUri, + textUnitDTOS); + meterRegistry + .counter( + "SmartlingSync.uploadAiTranslationsBatch", + "repository", + repository.getName(), + "jsonSync", + "true") + .increment(textUnitDTOS.size()); + } + }); + }); + } + } + + private void uploadLocalizedFile( + String projectId, + Map localeMapping, + RepositoryLocale repositoryLocale, + String uploadedFileUri, + List textUnitDTOS) { + String fileContent = + smartlingJsonConverter.textUnitDTOsToJsonString(textUnitDTOS, TextUnitDTO::getTarget); + String smartlingLocale = getSmartlingLocale(localeMapping, repositoryLocale); + + Mono.fromCallable( + () -> + smartlingClient.uploadLocalizedFile( + projectId, uploadedFileUri, "json", smartlingLocale, fileContent, null, null)) + .retryWhen( + smartlingClient + .getRetryConfiguration() + .doBeforeRetry( + e -> + logger.info( + String.format( + "Retrying after failure to upload localized file %s in project %s", + uploadedFileUri, projectId), + e.failure()))) + .doOnError( + e -> { + String msg = + String.format( + "Error uploading localized file %s in project %s", + uploadedFileUri, projectId); + logger.error(msg, e); + throw new SmartlingClientException(msg, e); + }) + .block(); + } + public void pushTranslations( Repository repository, String projectId, @@ -291,41 +392,12 @@ public void pushTranslations( .tag("repository", repository.getName())) { String fileName = getSourceFileName(repository.getName(), index); - String fileContent = - smartlingJsonConverter.textUnitDTOsToJsonString( - textUnitDTOS, TextUnitDTO::getTarget); - String smartlingLocale = - getSmartlingLocale(localeMapping, repositoryLocale); - Mono.fromCallable( - () -> - smartlingClient.uploadLocalizedFile( - projectId, - fileName, - "json", - smartlingLocale, - fileContent, - null, - null)) - .retryWhen( - smartlingClient - .getRetryConfiguration() - .doBeforeRetry( - e -> - logger.info( - String.format( - "Retrying after failure to upload localized file %s in project %s", - fileName, projectId), - e.failure()))) - .doOnError( - e -> { - String msg = - String.format( - "Error uploading localized file %s in project %s", - fileName, projectId); - logger.error(msg, e); - throw new SmartlingClientException(msg, e); - }) - .block(); + uploadLocalizedFile( + projectId, + localeMapping, + repositoryLocale, + fileName, + textUnitDTOS); return index; } }) @@ -353,6 +425,41 @@ PageFetcherOffsetAndLimitSplitIterator getSourceTextUnitIterator( parameters.setLimit(limit); parameters.setPluralFormsFiltered(true); parameters.setOrderByTextUnitID(true); + parameters.setExcludeUnexpiredPendingMT(aiTranslationConfiguration.isEnabled()); + parameters.setAiTranslationExpiryDuration( + aiTranslationConfiguration.getExpiryDuration()); + List search = textUnitSearcher.search(parameters); + return search; + }, + batchSize); + + return textUnitDTOPageFetcherOffsetAndLimitSplitIterator; + } + + PageFetcherOffsetAndLimitSplitIterator getTargetTextUnitIterator( + Repository repository, + Long localeId, + String skipTextUnitsWithPattern, + String skipAssetsWithPathPattern, + String includeTextUnitsWithPattern, + StatusFilter statusFilter) { + + PageFetcherOffsetAndLimitSplitIterator + textUnitDTOPageFetcherOffsetAndLimitSplitIterator = + new PageFetcherOffsetAndLimitSplitIterator<>( + (offset, limit) -> { + TextUnitSearcherParameters parameters = new TextUnitSearcherParameters(); + parameters.setRepositoryIds(repository.getId()); + parameters.setLocaleId(localeId); + parameters.setDoNotTranslateFilter(false); + parameters.setStatusFilter(statusFilter); + parameters.setUsedFilter(UsedFilter.USED); + parameters.setSkipTextUnitWithPattern(skipTextUnitsWithPattern); + parameters.setSkipAssetPathWithPattern(skipAssetsWithPathPattern); + parameters.setIncludeTextUnitsWithPattern(includeTextUnitsWithPattern); + parameters.setOffset(offset); + parameters.setLimit(limit); + parameters.setPluralFormsFiltered(true); List search = textUnitSearcher.search(parameters); return search; }, diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/SmartlingResultProcessor.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/SmartlingResultProcessor.java index e6df0dc871..e5da9c26dc 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/SmartlingResultProcessor.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/smartling/SmartlingResultProcessor.java @@ -37,6 +37,10 @@ public String processPushTranslations(List files, SmartlingOption return processAction(files, options, "push_translations"); } + public String processPushAiTranslations(List files, SmartlingOptions options) { + return processAction(files, options, "push_ai_translations"); + } + private String processAction(List files, SmartlingOptions options, String action) { String result = null; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/StatusFilter.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/StatusFilter.java index 6d1b0ba10c..c0d8baa316 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/StatusFilter.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/StatusFilter.java @@ -54,4 +54,6 @@ public enum StatusFilter { NOT_REJECTED, /** TextUnits with status ({@link TMTextUnitVariant.Status#OVERRIDDEN}). */ OVERRIDDEN, + /** TextUnits with status (@link TMTextUnitVariant.Status#MT_TRANSLATED}). */ + MT_TRANSLATED } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java index 025ad75599..2595a0ef2d 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java @@ -34,6 +34,7 @@ public class TextUnitDTO { private Long assetTextUnitId; private ZonedDateTime tmTextUnitCreatedDate; private boolean doNotTranslate; + private String uploadedFileUri; public Long getTmTextUnitId() { return tmTextUnitId; @@ -242,4 +243,12 @@ public boolean isDoNotTranslate() { public void setDoNotTranslate(boolean doNotTranslate) { this.doNotTranslate = doNotTranslate; } + + public String getUploadedFileUri() { + return uploadedFileUri; + } + + public void setUploadedFileUri(String uploadedFileUri) { + this.uploadedFileUri = uploadedFileUri; + } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java index dd2c35bd25..f414b23589 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java @@ -56,6 +56,10 @@ public TextUnitDTO mapObject(CriteriaResult cr) { String doNotTranslate = cr.getString(idx++); t.setDoNotTranslate(Boolean.valueOf(doNotTranslate)); + if (cr.hasProperty("uploadedFileUri")) { + t.setUploadedFileUri(cr.getString(idx++)); + } + return t; } diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java index b76335e6a1..18882d854e 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java @@ -165,6 +165,13 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters) "tm_text_unit_pending_mt", "tmtupmt", "tmtupmt.tm_text_unit_id", "tu.id")); } + if (searchParameters.shouldRetrieveUploadedFileUri()) { + // Retrieve the uploadedFileUri from the ThirdPartyTextUnit table (used when pushing AI + // translations to third party) + c.addJoin( + NativeExps.innerJoin("third_party_text_unit", "tptu", "tptu.tm_text_unit_id", "tu.id")); + } + NativeJunctionExp onClauseRepositoryLocale = NativeExps.conjunction(); onClauseRepositoryLocale.add(new NativeColumnEqExp("rl.locale_id", "l.id")); onClauseRepositoryLocale.add(new NativeColumnEqExp("rl.repository_id", "r.id")); @@ -212,15 +219,11 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters) "plural_form_for_locale", "pffl", NativeJoin.JoinType.LEFT_OUTER, onClausePluralForm)); logger.debug("Set projections"); - - // TODO(P1) Might want to some of those projection as optional for perf reason - c.setProjection( + NativeProjection projection = NativeExps.projection() .addProjection("tu.id", "tmTextUnitId") .addProjection("tuv.id", "tmTextUnitVariantId") - . - // TODO(PO) THIS NOT CONSISTANT !! chooose - addProjection("l.id", "localeId") + .addProjection("l.id", "localeId") .addProjection("l.bcp47_tag", "targetLocale") .addProjection("tu.name", "name") .addProjection("tu.content", "source") @@ -242,7 +245,12 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters) .addProjection("a.path", "assetPath") .addProjection("atu.id", "assetTextUnitId") .addProjection("tu.created_date", "tmTextUnitCreatedDate") - .addProjection("atu.do_not_translate", "doNotTranslate")); + .addProjection("atu.do_not_translate", "doNotTranslate"); + + if (searchParameters.shouldRetrieveUploadedFileUri()) { + projection.addProjection("tptu.uploaded_file_uri", "uploadedFileUri"); + } + c.setProjection(projection); logger.debug("Add search filters"); NativeJunctionExp conjunction = NativeExps.conjunction(); @@ -408,6 +416,10 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters) new NativeEqExpFix( "tuv.status", TMTextUnitVariant.Status.TRANSLATION_NEEDED.toString())); break; + case MT_TRANSLATED: + conjunction.add( + new NativeEqExpFix("tuv.status", TMTextUnitVariant.Status.MT_TRANSLATED.toString())); + break; case TRANSLATED: conjunction.add(NativeExps.isNotNull("tuv.id")); break; diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java index 8825b48ba0..b35f510537 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java +++ b/webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java @@ -53,6 +53,7 @@ public class TextUnitSearcherParameters { String skipAssetPathWithPattern; boolean isExcludeUnexpiredPendingMT = false; Duration aiTranslationExpiryDuration; + boolean shouldRetrieveUploadedFileUri = false; public String getName() { return name; @@ -347,6 +348,14 @@ public void setAiTranslationExpiryDuration(Duration aiTranslationExpiryDuration) this.aiTranslationExpiryDuration = aiTranslationExpiryDuration; } + public boolean shouldRetrieveUploadedFileUri() { + return shouldRetrieveUploadedFileUri; + } + + public void setIsRetrieveUploadedFileUri(boolean retrieveUploadedFileUri) { + this.shouldRetrieveUploadedFileUri = retrieveUploadedFileUri; + } + public static class Builder { private String name; private String source; @@ -383,6 +392,7 @@ public static class Builder { private String skipAssetPathWithPattern; private boolean isExcludeUnexpiredPendingMT; private Duration aiTranslationExpiryDuration; + private boolean shouldRetrieveUploadedFileUri; public TextUnitSearcherParameters build() { TextUnitSearcherParameters textUnitSearcherParameters = new TextUnitSearcherParameters(); @@ -421,6 +431,7 @@ public TextUnitSearcherParameters build() { textUnitSearcherParameters.skipAssetPathWithPattern = this.skipAssetPathWithPattern; textUnitSearcherParameters.isExcludeUnexpiredPendingMT = this.isExcludeUnexpiredPendingMT; textUnitSearcherParameters.aiTranslationExpiryDuration = this.aiTranslationExpiryDuration; + textUnitSearcherParameters.shouldRetrieveUploadedFileUri = this.shouldRetrieveUploadedFileUri; return textUnitSearcherParameters; } @@ -613,5 +624,10 @@ public Builder aiTranslationExpiryDuration(Duration aiTranslationExpiryDuration) this.aiTranslationExpiryDuration = aiTranslationExpiryDuration; return this; } + + public Builder shouldRetrieveUploadedFileUri(boolean shouldRetrieveUploadedFileUri) { + this.shouldRetrieveUploadedFileUri = shouldRetrieveUploadedFileUri; + return this; + } } } diff --git a/webapp/src/main/java/com/box/l10n/mojito/smartling/SmartlingClient.java b/webapp/src/main/java/com/box/l10n/mojito/smartling/SmartlingClient.java index 83719e464b..909343592e 100644 --- a/webapp/src/main/java/com/box/l10n/mojito/smartling/SmartlingClient.java +++ b/webapp/src/main/java/com/box/l10n/mojito/smartling/SmartlingClient.java @@ -369,6 +369,26 @@ public FileUploadResponse uploadLocalizedFile( String fileContent, String placeholderFormat, String placeholderFormatCustom) { + return uploadLocalizedFile( + projectId, + fileUri, + fileType, + localeId, + fileContent, + placeholderFormat, + placeholderFormatCustom, + "PUBLISHED"); + } + + public FileUploadResponse uploadLocalizedFile( + String projectId, + String fileUri, + String fileType, + String localeId, + String fileContent, + String placeholderFormat, + String placeholderFormatCustom, + String translationState) { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { HttpPost uploadFileMethod = @@ -387,7 +407,7 @@ public FileUploadResponse uploadLocalizedFile( multipartEntityBuilder.addTextBody("fileUri", fileUri); multipartEntityBuilder.addTextBody("fileType", fileType); - multipartEntityBuilder.addTextBody("translationState", "PUBLISHED"); + multipartEntityBuilder.addTextBody("translationState", translationState); multipartEntityBuilder.addTextBody("overwrite", "true"); if (!Strings.isNullOrEmpty(placeholderFormat)) { multipartEntityBuilder.addTextBody("smartling.placeholder_format", placeholderFormat); diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/ai/translation/AITranslateCronJobTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/ai/translation/AITranslateCronJobTest.java index 7f40786d40..57028fb310 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/ai/translation/AITranslateCronJobTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/ai/translation/AITranslateCronJobTest.java @@ -40,6 +40,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Queue; import java.util.Set; import org.assertj.core.util.Lists; import org.junit.Before; @@ -368,7 +369,7 @@ public void testBatchLogic() throws JobExecutionException { assertEquals(3, aiTranslations.size()); assertThat(aiTranslations).extracting("localeId").containsExactlyInAnyOrder(2L, 3L, 4L); } - verify(aiTranslationService, times(10)).sendForDeletion(isA(TmTextUnitPendingMT.class)); + verify(aiTranslationService, times(3)).deleteBatch(isA(Queue.class)); } @Test @@ -403,7 +404,7 @@ public void testBatchRequestFailLogic() throws JobExecutionException { assertEquals(3, aiTranslations.size()); assertThat(aiTranslations).extracting("localeId").containsExactlyInAnyOrder(2L, 3L, 4L); } - verify(aiTranslationService, times(10)).sendForDeletion(isA(TmTextUnitPendingMT.class)); + verify(aiTranslationService, times(3)).deleteBatch(isA(Queue.class)); } @Test @@ -437,6 +438,6 @@ public void testBatchLogicFailureToRetrieveTextUnit() throws JobExecutionExcepti assertEquals(3, aiTranslations.size()); assertThat(aiTranslations).extracting("localeId").containsExactlyInAnyOrder(2L, 3L, 4L); } - verify(aiTranslationService, times(5)).sendForDeletion(isA(TmTextUnitPendingMT.class)); + verify(aiTranslationService, times(2)).deleteBatch(isA(Queue.class)); } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/StubSmartlingResultProcessor.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/StubSmartlingResultProcessor.java index 1784fb21ca..7f7a6fe225 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/StubSmartlingResultProcessor.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/StubSmartlingResultProcessor.java @@ -10,6 +10,7 @@ public class StubSmartlingResultProcessor extends SmartlingResultProcessor { List pushFiles = new ArrayList<>(); List pushTranslationFiles = new ArrayList<>(); + List pushAITranslationFiles = new ArrayList<>(); SmartlingOptions options; public StubSmartlingResultProcessor() {} @@ -27,4 +28,11 @@ public String processPushTranslations(List files, SmartlingOption this.options = options; return ""; } + + @Override + public String processPushAiTranslations(List files, SmartlingOptions options) { + this.pushAITranslationFiles = files; + this.options = options; + return ""; + } } diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java index 0d29583fae..863eb01f2e 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java @@ -30,11 +30,14 @@ import com.box.l10n.mojito.entity.Repository; import com.box.l10n.mojito.entity.TM; import com.box.l10n.mojito.entity.TMTextUnit; +import com.box.l10n.mojito.entity.TMTextUnitVariant; +import com.box.l10n.mojito.entity.ThirdPartyTextUnit; import com.box.l10n.mojito.json.ObjectMapper; import com.box.l10n.mojito.quartz.QuartzJobInfo; import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler; import com.box.l10n.mojito.quartz.QuartzSchedulerManager; import com.box.l10n.mojito.service.ai.translation.AITranslationConfiguration; +import com.box.l10n.mojito.service.ai.translation.AITranslationService; import com.box.l10n.mojito.service.asset.AssetService; import com.box.l10n.mojito.service.assetExtraction.AssetExtractionRepository; import com.box.l10n.mojito.service.assetExtraction.AssetExtractionService; @@ -161,14 +164,20 @@ public class ThirdPartyTMSSmartlingTest extends ServiceTestBase { @Autowired QuartzSchedulerManager schedulerManager; + @Autowired ThirdPartyTextUnitRepository thirdPartyTextUnitRepository; + StubSmartlingResultProcessor resultProcessor; @Mock TextUnitBatchImporterService mockTextUnitBatchImporterService; @Mock AITranslationConfiguration aiTranslationConfiguration; + @Mock AITranslationService aiTranslationService; + @Captor ArgumentCaptor> textUnitListCaptor; + @Captor ArgumentCaptor> variantIdsCaptor; + @Captor ArgumentCaptor> quartzJobInfoCaptor; @@ -207,6 +216,7 @@ public void setUp() throws SchedulerException { anyString(), anyString(), anyString(), + anyString(), anyString()); doReturn(null) .when(mockTextUnitBatchImporterService) @@ -225,7 +235,8 @@ public void setUp() throws SchedulerException { assetTextUnitToTMTextUnitRepository, meterRegistry, mockQuartzPollableTaskScheduler, - aiTranslationConfiguration); + aiTranslationConfiguration, + aiTranslationService); mapper = new AndroidStringDocumentMapper(pluralSep, null); RetryBackoffSpec retryConfiguration = @@ -302,7 +313,8 @@ public void testPushInBatchesWithSingularsAndNoPlurals() batchSize, meterRegistry, mockQuartzPollableTaskScheduler, - aiTranslationConfiguration); + aiTranslationConfiguration, + aiTranslationService); TM tm = repository.getTm(); Asset asset = @@ -357,7 +369,8 @@ public void testRetryDuringPush() throws RepositoryNameAlreadyUsedException { batchSize, meterRegistry, mockQuartzPollableTaskScheduler, - aiTranslationConfiguration); + aiTranslationConfiguration, + aiTranslationService); // throw timeout exception for first request, following request should be successful when(smartlingClient.uploadFile(any(), any(), any(), any(), any(), any(), any())) .thenThrow( @@ -424,7 +437,8 @@ public void testRetriesExhaustedDuringPush() throws RepositoryNameAlreadyUsedExc batchSize, meterRegistry, mockQuartzPollableTaskScheduler, - aiTranslationConfiguration); + aiTranslationConfiguration, + aiTranslationService); TM tm = repository.getTm(); Asset asset = @@ -476,7 +490,8 @@ public void testPushInBatchesWithNoSingularsAndPlurals() batchSize, meterRegistry, mockQuartzPollableTaskScheduler, - aiTranslationConfiguration); + aiTranslationConfiguration, + aiTranslationService); TM tm = repository.getTm(); Asset asset = @@ -536,7 +551,8 @@ public void testPushInBatchesWithSingularsAndPlurals() throws RepositoryNameAlre batchSize, meterRegistry, mockQuartzPollableTaskScheduler, - aiTranslationConfiguration); + aiTranslationConfiguration, + aiTranslationService); TM tm = repository.getTm(); Asset asset = @@ -752,7 +768,8 @@ public void testPullNoBatches() throws RepositoryLocaleCreationException, Interr assetTextUnitToTMTextUnitRepository, meterRegistry, mockQuartzPollableTaskScheduler, - aiTranslationConfiguration); + aiTranslationConfiguration, + aiTranslationService); tmsSmartling.pull( repository, "projectId", @@ -825,7 +842,8 @@ public void testPullNoBatchesPluralFix() throws RepositoryLocaleCreationExceptio assetTextUnitToTMTextUnitRepository, meterRegistry, mockQuartzPollableTaskScheduler, - aiTranslationConfiguration); + aiTranslationConfiguration, + aiTranslationService); tmsSmartling.pull( repository, "projectId", @@ -889,7 +907,8 @@ public void testPullDryRunNoBatches() throws RepositoryLocaleCreationException { assetTextUnitToTMTextUnitRepository, meterRegistry, mockQuartzPollableTaskScheduler, - aiTranslationConfiguration); + aiTranslationConfiguration, + aiTranslationService); tmsSmartling.pull( repository, "projectId", @@ -983,7 +1002,8 @@ public void testPushTranslationsSingularsNoBatches() eq(locale.getBcp47Tag()), startsWith(" result; Repository repository = repositoryService.createRepository(testIdWatcher.getEntityName("batchRepo")); @@ -1043,7 +1070,14 @@ public void testRetriesExhaustedPushTranslations() verify(smartlingClient, times(11)) .uploadLocalizedFile( - anyString(), anyString(), anyString(), anyString(), anyString(), any(), any()); + anyString(), + anyString(), + anyString(), + anyString(), + anyString(), + any(), + any(), + eq("PUBLISHED")); } @Test @@ -1051,7 +1085,14 @@ public void testRetryPushTranslations() throws RepositoryNameAlreadyUsedException, RepositoryLocaleCreationException { // First request results in timeout, retry then successful when(smartlingClient.uploadLocalizedFile( - anyString(), anyString(), anyString(), anyString(), anyString(), any(), any())) + anyString(), + anyString(), + anyString(), + anyString(), + anyString(), + any(), + any(), + anyString())) .thenThrow( new SmartlingClientException(new HttpServerErrorException(HttpStatus.GATEWAY_TIMEOUT))) .thenReturn(null); @@ -1124,7 +1165,14 @@ public void testRetryPushTranslations() // Verify upload localized file called three times due to retry on first request verify(smartlingClient, times(3)) .uploadLocalizedFile( - anyString(), anyString(), anyString(), anyString(), anyString(), eq(null), eq(null)); + anyString(), + anyString(), + anyString(), + anyString(), + anyString(), + eq(null), + eq(null), + eq("PUBLISHED")); } @Test @@ -1327,7 +1375,8 @@ public void testPushTranslationsNoBatches() eq(locale.getBcp47Tag()), startsWith(" result; + Repository repository = + repositoryService.createRepository(testIdWatcher.getEntityName("pushAiTranslationsRepo")); + Locale frCA = localeService.findByBcp47Tag("fr-CA"); + Locale jaJP = localeService.findByBcp47Tag("ja-JP"); + repositoryService.addRepositoryLocale(repository, frCA.getBcp47Tag()); + repositoryService.addRepositoryLocale(repository, jaJP.getBcp47Tag()); + + PluralForm one = pluralFormService.findByPluralFormString("one"); + + TM tm = repository.getTm(); + Asset asset = + assetService.createAssetWithContent(repository.getId(), "fake_for_test", "fake for test"); + AssetExtraction assetExtraction = new AssetExtraction(); + assetExtraction.setAsset(asset); + assetExtraction = assetExtractionRepository.save(assetExtraction); + + int textUnits = 5; + + for (int i = 0; i < textUnits; i++) { + String name = "singular_message" + i; + String content = "Singular Message Test #" + i; + String comment = "Singular Comment" + i; + TMTextUnit textUnit = + tmService.addTMTextUnit(tm.getId(), asset.getId(), name, content, comment); + tmService.addTMTextUnitCurrentVariant( + textUnit.getId(), + frCA.getId(), + String.format("%s in %s", content, frCA.getBcp47Tag()), + "test comment", + TMTextUnitVariant.Status.MT_TRANSLATED); + tmService.addTMTextUnitCurrentVariant( + textUnit.getId(), + jaJP.getId(), + String.format("%s in %s", content, jaJP.getBcp47Tag()), + "test comment", + TMTextUnitVariant.Status.MT_TRANSLATED); + assetExtractionService.createAssetTextUnit(assetExtraction, name, content, comment); + + ThirdPartyTextUnit thirdPartyTextUnit = new ThirdPartyTextUnit(); + thirdPartyTextUnit.setTmTextUnit(textUnit); + thirdPartyTextUnit.setAsset(asset); + thirdPartyTextUnit.setUploadedFileUri("testSingularUploadedFileUri"); + thirdPartyTextUnitRepository.save(thirdPartyTextUnit); + } + + for (int i = 0; i < textUnits; i++) { + ZonedDateTime now = ZonedDateTime.now(); + String name = "plural_message" + i; + String content = "Plural Message Test #" + i; + String comment = "Plural Comment" + i; + String pluralFormOther = "plural_form_other" + i; + + TMTextUnit tu = + tmService.addTMTextUnit(tm, asset, name, content, comment, now, one, pluralFormOther); + assetExtractionService.createAssetTextUnit(assetExtraction, name, content, comment); + + tmService.addTMTextUnitCurrentVariant( + tu.getId(), + frCA.getId(), + String.format("%s in %s", content, frCA.getBcp47Tag()), + "test comment", + TMTextUnitVariant.Status.MT_TRANSLATED); + tmService.addTMTextUnitCurrentVariant( + tu.getId(), + jaJP.getId(), + String.format("%s in %s", content, jaJP.getBcp47Tag()), + "test comment", + TMTextUnitVariant.Status.MT_TRANSLATED); + + ThirdPartyTextUnit thirdPartyTextUnit = new ThirdPartyTextUnit(); + thirdPartyTextUnit.setTmTextUnit(tu); + thirdPartyTextUnit.setAsset(asset); + thirdPartyTextUnit.setUploadedFileUri("testPluralUploadedFileUri"); + thirdPartyTextUnitRepository.save(thirdPartyTextUnit); + } + + prepareAssetAndTextUnits(assetExtraction, asset, tm); + + tmsSmartling.pushAITranslations( + repository, + "projectId", + pluralSep, + ImmutableMap.of(), + null, + null, + null, + ImmutableList.of()); + result = resultProcessor.pushAITranslationFiles; + + assertThat(result).hasSize(4); + Stream.of(jaJP, frCA) + .forEach( + locale -> { + SmartlingFile singularFile = + result.stream() + .filter( + f -> + f.getFileName() + .matches("testSingularUploadedFileUri_" + locale.getBcp47Tag())) + .findFirst() + .get(); + + SmartlingFile pluralFile = + result.stream() + .filter( + f -> + f.getFileName() + .matches("testPluralUploadedFileUri_" + locale.getBcp47Tag())) + .findFirst() + .get(); + + assertThat(singularFile.getFileName()) + .isEqualTo("testSingularUploadedFileUri_" + locale.getBcp47Tag()); + List singulars = readTextUnits(singularFile, pluralSep); + assertThat(singulars) + .allSatisfy( + tu -> { + assertThat(tu.getComment()).startsWith("Singular Comment"); + assertThat(tu.getName()).startsWith("singular_message"); + assertThat(tu.getAssetPath()).isEqualTo("fake_for_test"); + assertThat(tu.getTarget()) + .matches("Singular Message Test #\\d in " + locale.getBcp47Tag()); + }); + + assertThat(pluralFile.getFileName()) + .isEqualTo("testPluralUploadedFileUri_" + locale.getBcp47Tag()); + List plurals = readTextUnits(pluralFile, pluralSep); + assertThat(plurals).hasSize(textUnits); + assertThat(plurals) + .allSatisfy( + tu -> { + assertThat(tu.getName()).endsWith("_one"); + assertThat(tu.getPluralForm()).isEqualTo("one"); + assertThat(tu.getTarget()) + .matches("Plural Message Test #\\d in " + locale.getBcp47Tag()); + }); + + verify(smartlingClient, times(1)) + .uploadLocalizedFile( + eq("projectId"), + eq("testSingularUploadedFileUri"), + eq("android"), + eq(locale.getBcp47Tag()), + startsWith(" result; + Repository repository = + repositoryService.createRepository(testIdWatcher.getEntityName("pushAiTranslationsRepo")); + Locale frCA = localeService.findByBcp47Tag("fr-CA"); + Locale jaJP = localeService.findByBcp47Tag("ja-JP"); + repositoryService.addRepositoryLocale(repository, frCA.getBcp47Tag()); + repositoryService.addRepositoryLocale(repository, jaJP.getBcp47Tag()); + + PluralForm one = pluralFormService.findByPluralFormString("one"); + + TM tm = repository.getTm(); + Asset asset = + assetService.createAssetWithContent(repository.getId(), "fake_for_test", "fake for test"); + AssetExtraction assetExtraction = new AssetExtraction(); + assetExtraction.setAsset(asset); + assetExtraction = assetExtractionRepository.save(assetExtraction); + + int textUnits = 5; + + for (int i = 0; i < textUnits; i++) { + String name = "singular_message" + i; + String content = "Singular Message Test #" + i; + String comment = "Singular Comment" + i; + TMTextUnit textUnit = + tmService.addTMTextUnit(tm.getId(), asset.getId(), name, content, comment); + tmService.addTMTextUnitCurrentVariant( + textUnit.getId(), + frCA.getId(), + String.format("%s in %s", content, frCA.getBcp47Tag()), + "test comment", + TMTextUnitVariant.Status.MT_TRANSLATED); + tmService.addTMTextUnitCurrentVariant( + textUnit.getId(), + jaJP.getId(), + String.format("%s in %s", content, jaJP.getBcp47Tag()), + "test comment", + TMTextUnitVariant.Status.MT_TRANSLATED); + assetExtractionService.createAssetTextUnit(assetExtraction, name, content, comment); + + ThirdPartyTextUnit thirdPartyTextUnit = new ThirdPartyTextUnit(); + thirdPartyTextUnit.setTmTextUnit(textUnit); + thirdPartyTextUnit.setAsset(asset); + thirdPartyTextUnit.setUploadedFileUri("testSingularUploadedFileUri_" + i); + thirdPartyTextUnitRepository.save(thirdPartyTextUnit); + } + + for (int i = 0; i < textUnits; i++) { + ZonedDateTime now = ZonedDateTime.now(); + String name = "plural_message" + i; + String content = "Plural Message Test #" + i; + String comment = "Plural Comment" + i; + String pluralFormOther = "plural_form_other" + i; + + TMTextUnit tu = + tmService.addTMTextUnit(tm, asset, name, content, comment, now, one, pluralFormOther); + assetExtractionService.createAssetTextUnit(assetExtraction, name, content, comment); + + tmService.addTMTextUnitCurrentVariant( + tu.getId(), + frCA.getId(), + String.format("%s in %s", content, frCA.getBcp47Tag()), + "test comment", + TMTextUnitVariant.Status.MT_TRANSLATED); + tmService.addTMTextUnitCurrentVariant( + tu.getId(), + jaJP.getId(), + String.format("%s in %s", content, jaJP.getBcp47Tag()), + "test comment", + TMTextUnitVariant.Status.MT_TRANSLATED); + + ThirdPartyTextUnit thirdPartyTextUnit = new ThirdPartyTextUnit(); + thirdPartyTextUnit.setTmTextUnit(tu); + thirdPartyTextUnit.setAsset(asset); + thirdPartyTextUnit.setUploadedFileUri("testPluralUploadedFileUri_" + i); + thirdPartyTextUnitRepository.save(thirdPartyTextUnit); + } + + prepareAssetAndTextUnits(assetExtraction, asset, tm); + + tmsSmartling.pushAITranslations( + repository, + "projectId", + pluralSep, + ImmutableMap.of(), + null, + null, + null, + ImmutableList.of()); + result = resultProcessor.pushAITranslationFiles; + + assertThat(result).hasSize(20); + Stream.of(jaJP, frCA) + .forEach( + locale -> { + SmartlingFile singularFile = + result.stream() + .filter( + f -> + f.getFileName().startsWith("testSingularUploadedFileUri_") + && f.getFileName().endsWith(locale.getBcp47Tag())) + .findFirst() + .get(); + + SmartlingFile pluralFile = + result.stream() + .filter( + f -> + f.getFileName().startsWith("testPluralUploadedFileUri_") + && f.getFileName().endsWith(locale.getBcp47Tag())) + .findFirst() + .get(); + + assertThat(singularFile.getFileName()) + .containsPattern("testSingularUploadedFileUri_[0-4]_" + locale.getBcp47Tag()); + assertThat(readTextUnits(singularFile, pluralSep)) + .allSatisfy( + tu -> { + assertThat(tu.getComment()).startsWith("Singular Comment"); + assertThat(tu.getName()).startsWith("singular_message"); + assertThat(tu.getAssetPath()).isEqualTo("fake_for_test"); + assertThat(tu.getTarget()) + .matches("Singular Message Test #\\d in " + locale.getBcp47Tag()); + }); + + assertThat(pluralFile.getFileName()) + .containsPattern("testPluralUploadedFileUri_[0-4]_" + locale.getBcp47Tag()); + List plurals = readTextUnits(pluralFile, pluralSep); + assertThat(plurals).hasSize(1); + assertThat(plurals) + .allSatisfy( + tu -> { + assertThat(tu.getName()).endsWith("_one"); + assertThat(tu.getPluralForm()).isEqualTo("one"); + assertThat(tu.getTarget()) + .matches("Plural Message Test #\\d in " + locale.getBcp47Tag()); + }); + for (int i = 0; i < textUnits; i++) { + verify(smartlingClient, times(1)) + .uploadLocalizedFile( + eq("projectId"), + eq("testSingularUploadedFileUri_" + i), + eq("android"), + eq(locale.getBcp47Tag()), + startsWith(" searchTextUnits(List ids) { TextUnitSearcherParameters params = new TextUnitSearcherParameters(); params.setTmTextUnitIds(ids); diff --git a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java index 636f45b2bb..b09391fc90 100644 --- a/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java +++ b/webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingWithJsonTest.java @@ -7,6 +7,7 @@ import com.box.l10n.mojito.entity.*; import com.box.l10n.mojito.json.ObjectMapper; +import com.box.l10n.mojito.service.ai.translation.AITranslationConfiguration; import com.box.l10n.mojito.service.asset.AssetService; import com.box.l10n.mojito.service.assetExtraction.AssetExtractionService; import com.box.l10n.mojito.service.assetExtraction.ServiceTestBase; @@ -83,6 +84,8 @@ public class ThirdPartyTMSSmartlingWithJsonTest extends ServiceTestBase { @Mock ThirdPartyFileChecksumRepository thirdPartyFileChecksumRepositoryMock; + @Mock AITranslationConfiguration aiTranslationConfiguration; + SmartlingJsonConverter smartlingJsonConverter = new SmartlingJsonConverter(ObjectMapper.withIndentedOutput(), new SmartlingJsonKeys()); @@ -255,7 +258,14 @@ public void testGetTranslatedUnits() { ThirdPartyTMSSmartlingWithJson thirdPartyTMSSmartlingWithJson = new ThirdPartyTMSSmartlingWithJson( - null, null, null, null, null, meterRegistryMock, thirdPartyFileChecksumRepositoryMock); + null, + null, + null, + null, + null, + meterRegistryMock, + thirdPartyFileChecksumRepositoryMock, + aiTranslationConfiguration); ImmutableList result = thirdPartyTMSSmartlingWithJson.getTranslatedUnits(