Added PUSH_AI_TRANSLATION to third party sync (#166)

* Batch delete in loop iteration * ThirdPartySync updates compiling * AI translation import to Smartling via POST_TRANSLATION upload * Added push ai translations support to json sync * Added unit tests for third party ai translation push * Remove dead code * Code review updates * Update variant status name * Documentation and renamed variable
pinterest · Nov 5, 2024 · 9fbd944 · 9fbd944
1 parent 1e764d6
commit 9fbd944
Show file tree

Hide file tree

Showing 21 changed files with 1,049 additions and 108 deletions.
diff --git a/restclient/src/main/java/com/box/l10n/mojito/rest/ThirdPartySyncAction.java b/restclient/src/main/java/com/box/l10n/mojito/rest/ThirdPartySyncAction.java
@@ -6,5 +6,6 @@ public enum ThirdPartySyncAction {
   PULL,
   PULL_SOURCE,
   MAP_TEXTUNIT,
-  PUSH_SCREENSHOT
+  PUSH_SCREENSHOT,
+  PUSH_AI_TRANSLATION,
 }
diff --git a/webapp/src/main/java/com/box/l10n/mojito/entity/TMTextUnitVariant.java b/webapp/src/main/java/com/box/l10n/mojito/entity/TMTextUnitVariant.java
@@ -69,9 +69,14 @@ public enum Status {
      */
     REVIEW_NEEDED,
 
+    /** Indicates that the text unit has been machine translated in Mojito automatically. */
     MT_TRANSLATED,
 
-    MT_REVIEW,
+    /**
+     * Indicates that the text unit has been machine translated in Mojito automatically and has been
+     * sent for third party review.
+     */
+    MT_REVIEW_NEEDED,
     /** A string that doesn't need any work to be performed on it. */
     APPROVED,
     /** It was overridden in Mojito, so it won't be updated during third-party sync */

diff --git a/webapp/src/main/java/com/box/l10n/mojito/rest/thirdparty/ThirdPartySyncAction.java b/webapp/src/main/java/com/box/l10n/mojito/rest/thirdparty/ThirdPartySyncAction.java
@@ -6,5 +6,6 @@ public enum ThirdPartySyncAction {
   PULL,
   PULL_SOURCE,
   MAP_TEXTUNIT,
-  PUSH_SCREENSHOT
+  PUSH_SCREENSHOT,
+  PUSH_AI_TRANSLATION
 }
diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/ai/translation/AITranslateCronJob.java b/webapp/src/main/java/com/box/l10n/mojito/service/ai/translation/AITranslateCronJob.java
@@ -21,8 +21,10 @@
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
+import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
@@ -271,6 +273,7 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution
         pendingMTs =
             tmTextUnitPendingMTRepository.findBatch(aiTranslationConfiguration.getBatchSize());
         logger.info("Processing {} pending MTs", pendingMTs.size());
+        Queue<TmTextUnitPendingMT> textUnitsToClearPendingMT = new ConcurrentLinkedQueue<>();
 
         List<CompletableFuture<Void>> futures =
             pendingMTs.stream()
@@ -295,7 +298,7 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution
                                   logger.debug(
                                       "Sending pending MT for tmTextUnitId: {} for deletion",
                                       pendingMT.getTmTextUnitId());
-                                  aiTranslationService.sendForDeletion(pendingMT);
+                                  textUnitsToClearPendingMT.add(pendingMT);
                                 }
                               }
                             },
@@ -304,6 +307,7 @@ public void execute(JobExecutionContext jobExecutionContext) throws JobExecution
 
         // Wait for all tasks in this batch to complete
         CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join();
+        aiTranslationService.deleteBatch(textUnitsToClearPendingMT);
       } while (!pendingMTs.isEmpty());
     } finally {
       shutdownExecutor(executorService);

diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/ai/translation/AITranslationService.java b/webapp/src/main/java/com/box/l10n/mojito/service/ai/translation/AITranslationService.java
@@ -1,30 +1,32 @@
 package com.box.l10n.mojito.service.ai.translation;
 
+import static com.box.l10n.mojito.entity.TMTextUnitVariant.Status.MT_REVIEW_NEEDED;
+
 import com.box.l10n.mojito.JSR310Migration;
 import com.box.l10n.mojito.entity.PromptType;
 import com.box.l10n.mojito.entity.TmTextUnitPendingMT;
 import com.box.l10n.mojito.service.ai.RepositoryLocaleAIPromptRepository;
 import com.box.l10n.mojito.service.repository.RepositoryRepository;
 import com.box.l10n.mojito.service.tm.TMTextUnitVariantRepository;
 import com.google.common.collect.Lists;
-import jakarta.annotation.PostConstruct;
-import jakarta.annotation.PreDestroy;
 import jakarta.transaction.Transactional;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
 import java.sql.Timestamp;
 import java.time.Duration;
 import java.util.List;
 import java.util.Map;
+import java.util.Queue;
 import java.util.Set;
 import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.jdbc.core.BatchPreparedStatementSetter;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Component;
-import reactor.core.publisher.Flux;
-import reactor.core.publisher.Sinks;
 
 @Component
 @ConditionalOnProperty(value = "l10n.ai.translation.enabled", havingValue = "true")
@@ -51,9 +53,6 @@ public class AITranslationService {
   @Value("${l10n.ai.translation.maxTextUnitsAIRequest:1000}")
   int maxTextUnitsAIRequest;
 
-  private final Sinks.Many<TmTextUnitPendingMT> pendingMTDeletionSink =
-      Sinks.many().multicast().onBackpressureBuffer();
-
   @Transactional
   public void createPendingMTEntitiesInBatches(Long repositoryId, Set<Long> tmTextUnitIds) {
     if (tmTextUnitIds.size() > maxTextUnitsAIRequest) {
@@ -72,9 +71,40 @@ public void createPendingMTEntitiesInBatches(Long repositoryId, Set<Long> tmText
     }
   }
 
-  protected void sendForDeletion(TmTextUnitPendingMT pendingMT) {
-    logger.debug("Sending pending MT for deletion: {}", pendingMT);
-    pendingMTDeletionSink.tryEmitNext(pendingMT);
+  @Transactional
+  public void updateVariantStatusToMTReview(List<Long> currentVariantIds) {
+
+    for (int i = 0; i < currentVariantIds.size(); i += batchSize) {
+      logger.debug("Updating variant statuses to MT_REVIEW in batches of {}", batchSize);
+      int end = Math.min(i + batchSize, currentVariantIds.size());
+      List<Long> updateBatch = currentVariantIds.subList(i, end);
+      executeVariantStatusUpdatesToMTReview(updateBatch);
+    }
+  }
+
+  private void executeVariantStatusUpdatesToMTReview(List<Long> updateBatch) {
+    String sql =
+        "UPDATE tm_text_unit_variant "
+            + "SET status = ? "
+            + "WHERE id IN ("
+            + "SELECT tucv.tm_text_unit_variant_id "
+            + "FROM tm_text_unit_current_variant tucv "
+            + "WHERE tucv.id = ?)";
+
+    jdbcTemplate.batchUpdate(
+        sql,
+        new BatchPreparedStatementSetter() {
+          @Override
+          public void setValues(PreparedStatement ps, int i) throws SQLException {
+            ps.setString(1, MT_REVIEW_NEEDED.name());
+            ps.setLong(2, updateBatch.get(i));
+          }
+
+          @Override
+          public int getBatchSize() {
+            return updateBatch.size();
+          }
+        });
   }
 
   private void createPendingMTEntitiesInBatches(Set<Long> tmTextUnitIds) {
@@ -177,7 +207,7 @@ private void insertMultiRowTextUnitCurrentVariants(
   }
 
   @Transactional
-  private void deleteBatch(List<TmTextUnitPendingMT> batch) {
+  protected void deleteBatch(Queue<TmTextUnitPendingMT> batch) {
     if (batch.isEmpty()) {
       logger.debug("No pending MTs to delete");
       return;
@@ -201,18 +231,4 @@ private static TmTextUnitPendingMT createTmTextUnitPendingMT(Long tmTextUnitId)
     tmTextUnitPendingMT.setCreatedDate(JSR310Migration.newDateTimeEmptyCtor());
     return tmTextUnitPendingMT;
   }
-
-  @PostConstruct
-  public void init() {
-    Flux<TmTextUnitPendingMT> flux = pendingMTDeletionSink.asFlux();
-
-    flux.bufferTimeout(batchSize, timeout)
-        .filter(batch -> !batch.isEmpty())
-        .subscribe(this::deleteBatch);
-  }
-
-  @PreDestroy
-  public void destroy() {
-    pendingMTDeletionSink.tryEmitComplete();
-  }
 }
diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java
@@ -180,6 +180,18 @@ void syncMojitoWithThirdPartyTMS(
     if (actions.contains(ThirdPartySyncAction.PUSH_SCREENSHOT)) {
       uploadScreenshotsAndCreateMappings(repository, thirdPartyProjectId, currentTask);
     }
+    if (actions.contains(ThirdPartySyncAction.PUSH_AI_TRANSLATION)) {
+      pushAITranslations(
+          thirdPartyProjectId,
+          pluralSeparator,
+          localeMapping,
+          skipTextUnitsWithPattern,
+          skipAssetsWithPathPattern,
+          includeTextUnitsWithPattern,
+          options,
+          repository,
+          currentTask);
+    }
   }
 
   @Pollable(message = "Push source strings to third party service.")
@@ -290,6 +302,28 @@ void mapMojitoAndThirdPartyTextUnits(
             e -> mapThirdPartyTextUnitsToTextUnitDTOs(e.getKey(), e.getValue(), pluralSeparator));
   }
 
+  @Pollable(message = "Push AI translations to third party service.")
+  void pushAITranslations(
+      String thirdPartyProjectId,
+      String pluralSeparator,
+      String localeMapping,
+      String skipTextUnitsWithPattern,
+      String skipAssetsWithPathPattern,
+      String includeTextUnitsWithPattern,
+      List<String> options,
+      Repository repository,
+      @ParentTask PollableTask currentTask) {
+    thirdPartyTMS.pushAITranslations(
+        repository,
+        thirdPartyProjectId,
+        pluralSeparator,
+        parseLocaleMapping(localeMapping),
+        skipTextUnitsWithPattern,
+        skipAssetsWithPathPattern,
+        includeTextUnitsWithPattern,
+        options);
+  }
+
   void mapThirdPartyTextUnitsToTextUnitDTOs(
       Asset asset, List<ThirdPartyTextUnit> thirdPartyTextUnitsToMap, String pluralSeparator) {
     logger.debug("Map third party text units to text unit DTOs for asset: {}", asset.getId());

diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMS.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMS.java
@@ -96,4 +96,14 @@ void pullSource(
       String projectId,
       List<String> optionList,
       Map<String, String> localeMapping);
+
+  void pushAITranslations(
+      Repository repository,
+      String projectId,
+      String pluralSeparator,
+      Map<String, String> localeMapping,
+      String skipTextUnitsWithPattern,
+      String skipAssetsWithPathPattern,
+      String includeTextUnitsWithPattern,
+      List<String> optionList);
 }
diff --git a/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSInMemory.java b/webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSInMemory.java
@@ -117,4 +117,15 @@ public void pullSource(
       String projectId,
       List<String> optionList,
       Map<String, String> localeMapping) {}
+
+  @Override
+  public void pushAITranslations(
+      Repository repository,
+      String projectId,
+      String pluralSeparator,
+      Map<String, String> localeMapping,
+      String skipTextUnitsWithPattern,
+      String skipAssetsWithPathPattern,
+      String includeTextUnitsWithPattern,
+      List<String> optionList) {}
 }