Updated TextUnitSearcher to exclude AI translations based on duration…

… or null id, updated map step to record uploaded file uri in DB

webapp/src/main/java/com/box/l10n/mojito/entity/ThirdPartyTextUnit.java

@@ -41,6 +41,9 @@ public class ThirdPartyTextUnit extends AuditableEntity {
       foreignKey = @ForeignKey(name = "FK__THIRD_PARTY_TEXT_UNIT__TM_TEXT_UNIT__ID"))
   TMTextUnit tmTextUnit;
 
+  @Column(name = "uploaded_file_uri")
+  String uploadedFileUri;
+
   public String getThirdPartyId() {
     return thirdPartyId;
   }
@@ -64,4 +67,12 @@ public Asset getAsset() {
   public void setAsset(Asset asset) {
     this.asset = asset;
   }
+
+  public String getUploadedFileUri() {
+    return uploadedFileUri;
+  }
+
+  public void setUploadedFileUri(String uploadedFileUri) {
+    this.uploadedFileUri = uploadedFileUri;
+  }
 }

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyService.java

@@ -361,6 +361,8 @@ void saveMojitoToThirdPartyTextUnitMapping(
                             thirdPartyTextUnit.setThirdPartyId(
                                 thirdPartyTextUnitForMapping.getId());
                             thirdPartyTextUnit.setAsset(asset);
+                            thirdPartyTextUnit.setUploadedFileUri(
+                                thirdPartyTextUnitForMapping.getUploadedFileUri());
                             TMTextUnit tmTextUnit = tmTextUnitRepository.getOne(tmTextUnitId);
 
                             if (tmTextUnitAlreadySaved.containsKey(tmTextUnitId)) {

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java

@@ -12,6 +12,7 @@
 import com.box.l10n.mojito.entity.Repository;
 import com.box.l10n.mojito.quartz.QuartzJobInfo;
 import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler;
+import com.box.l10n.mojito.service.ai.translation.AITranslationConfiguration;
 import com.box.l10n.mojito.service.assetExtraction.AssetTextUnitToTMTextUnitRepository;
 import com.box.l10n.mojito.service.pollableTask.PollableFuture;
 import com.box.l10n.mojito.service.thirdparty.smartling.SmartlingFile;
@@ -42,6 +43,7 @@
 import io.micrometer.core.annotation.Timed;
 import io.micrometer.core.instrument.MeterRegistry;
 import io.micrometer.core.instrument.Timer;
+import java.time.Duration;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -100,6 +102,8 @@ public class ThirdPartyTMSSmartling implements ThirdPartyTMS {
 
   private final QuartzPollableTaskScheduler quartzPollableTaskScheduler;
 
+  private final AITranslationConfiguration aiTranslationConfiguration;
+
   private final Set<String> supportedImageExtensions =
       Sets.newHashSet("png", "jpg", "jpeg", "gif", "tiff");
 
@@ -118,7 +122,8 @@ public ThirdPartyTMSSmartling(
       ThirdPartyTMSSmartlingGlossary thirdPartyTMSSmartlingGlossary,
       AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository,
       MeterRegistry meterRegistry,
-      QuartzPollableTaskScheduler quartzPollableTaskScheduler) {
+      QuartzPollableTaskScheduler quartzPollableTaskScheduler,
+      AITranslationConfiguration aiTranslationConfiguration) {
     this(
         smartlingClient,
         textUnitSearcher,
@@ -130,7 +135,8 @@ public ThirdPartyTMSSmartling(
         assetTextUnitToTMTextUnitRepository,
         DEFAULT_BATCH_SIZE,
         meterRegistry,
-        quartzPollableTaskScheduler);
+        quartzPollableTaskScheduler,
+        aiTranslationConfiguration);
   }
 
   public ThirdPartyTMSSmartling(
@@ -144,7 +150,8 @@ public ThirdPartyTMSSmartling(
       AssetTextUnitToTMTextUnitRepository assetTextUnitToTMTextUnitRepository,
       int batchSize,
       MeterRegistry meterRegistry,
-      QuartzPollableTaskScheduler quartzPollableTaskScheduler) {
+      QuartzPollableTaskScheduler quartzPollableTaskScheduler,
+      AITranslationConfiguration aiTranslationConfiguration) {
     this.smartlingClient = smartlingClient;
     this.assetPathAndTextUnitNameKeys = assetPathAndTextUnitNameKeys;
     this.textUnitBatchImporterService = textUnitBatchImporterService;
@@ -156,6 +163,7 @@ public ThirdPartyTMSSmartling(
     this.assetTextUnitToTMTextUnitRepository = assetTextUnitToTMTextUnitRepository;
     this.meterRegistry = meterRegistry;
     this.quartzPollableTaskScheduler = quartzPollableTaskScheduler;
+    this.aiTranslationConfiguration = aiTranslationConfiguration;
   }
 
   @Override
@@ -337,6 +345,7 @@ public List<ThirdPartyTextUnit> getThirdPartyTextUnits(
                         thirdPartyTextUnit.setAssetPath(key.getAssetPath());
                         thirdPartyTextUnit.setName(key.getTextUnitName());
                         thirdPartyTextUnit.setNamePluralPrefix(isPluralFile(file.getFileUri()));
+                        thirdPartyTextUnit.setUploadedFileUri(file.getFileUri());
 
                         return thirdPartyTextUnit;
                       });
@@ -968,6 +977,12 @@ private TextUnitSearcherParameters baseParams(
     result.setPluralFormsExcluded(pluralFormsExcluded);
     result.setSkipTextUnitWithPattern(skipTextUnitsWithPattern);
     result.setSkipAssetPathWithPattern(skipAssetsWithPathPattern);
+    result.setExcludeUnexpiredPendingMT(
+        aiTranslationConfiguration != null && aiTranslationConfiguration.isEnabled());
+    result.setAiTranslationExpiryDuration(
+        aiTranslationConfiguration != null
+            ? aiTranslationConfiguration.getExpiryDuration()
+            : Duration.ofHours(3));
     if (!Strings.isNullOrEmpty(pluralFormOther)) {
       result.setPluralFormOther(pluralFormOther);
     }

webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTextUnit.java

@@ -30,6 +30,8 @@ public class ThirdPartyTextUnit implements TextUnitForBatchMatcher {
    */
   boolean namePluralPrefix;
 
+  String uploadedFileUri;
+
   public String getId() {
     return id;
   }
@@ -80,4 +82,12 @@ public boolean isNamePluralPrefix() {
   public void setNamePluralPrefix(boolean namePluralPrefix) {
     this.namePluralPrefix = namePluralPrefix;
   }
+
+  public String getUploadedFileUri() {
+    return uploadedFileUri;
+  }
+
+  public void setUploadedFileUri(String uploadedFileUri) {
+    this.uploadedFileUri = uploadedFileUri;
+  }
 }

webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java

@@ -24,6 +24,7 @@
 import com.github.pnowy.nc.core.expressions.NativeProjection;
 import com.github.pnowy.nc.core.mappers.CriteriaResultTransformer;
 import com.google.common.base.Preconditions;
+import java.time.ZonedDateTime;
 import java.util.Arrays;
 import java.util.List;
 import org.apache.commons.codec.digest.DigestUtils;
@@ -172,6 +173,11 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters)
     c.addJoin(NativeExps.crossJoin("locale", "l"));
     c.addJoin(NativeExps.innerJoin("asset", "a", "a.id", "tu.asset_id"));
     c.addJoin(NativeExps.innerJoin("repository", "r", "r.id", "a.repository_id"));
+    if (searchParameters.isExcludeUnexpiredPendingMT()) {
+      c.addJoin(
+          NativeExps.leftJoin(
+              "tm_text_unit_pending_mt", "tmtupmt", "tmtupmt.tm_text_unit_id", "tu.id"));
+    }
 
     NativeJunctionExp onClauseRepositoryLocale = NativeExps.conjunction();
     onClauseRepositoryLocale.add(new NativeColumnEqExp("rl.locale_id", "l.id"));
@@ -378,6 +384,18 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters)
           new NativeNotILikeExp("a.path", searchParameters.getSkipAssetPathWithPattern()));
     }
 
+    if (searchParameters.isExcludeUnexpiredPendingMT()) {
+      ZonedDateTime durationThreshold =
+          ZonedDateTime.now().minus(searchParameters.getAiTranslationExpiryDuration());
+
+      // Include rows where the id in the left table is null (i.e. no match) or the created_date is
+      // outside the configured expiry duration
+      NativeJunctionExp mtExclusionFilter = NativeExps.disjunction();
+      mtExclusionFilter.add(NativeExps.isNull("tmtupmt.id"));
+      mtExclusionFilter.add(new NativeDateLteExp("tmtupmt.created_date", durationThreshold));
+      conjunction.add(mtExclusionFilter);
+    }
+
     StatusFilter statusFilter = searchParameters.getStatusFilter();
 
     if (statusFilter != null) {

webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcherParameters.java

@@ -1,6 +1,7 @@
 package com.box.l10n.mojito.service.tm.search;
 
 import com.box.l10n.mojito.service.NormalizationUtils;
+import java.time.Duration;
 import java.time.ZonedDateTime;
 import java.util.Arrays;
 import java.util.List;
@@ -50,6 +51,8 @@ public class TextUnitSearcherParameters {
   String skipTextUnitWithPattern;
   String includeTextUnitsWithPattern;
   String skipAssetPathWithPattern;
+  boolean isExcludeUnexpiredPendingMT = false;
+  Duration aiTranslationExpiryDuration;
 
   public String getName() {
     return name;
@@ -327,4 +330,20 @@ public boolean isOrderedByTextUnitID() {
   public void setOrderByTextUnitID(boolean ordered) {
     isOrderedByTextUnitID = ordered;
   }
+
+  public boolean isExcludeUnexpiredPendingMT() {
+    return isExcludeUnexpiredPendingMT;
+  }
+
+  public void setExcludeUnexpiredPendingMT(boolean excludeUnexpiredPendingMT) {
+    isExcludeUnexpiredPendingMT = excludeUnexpiredPendingMT;
+  }
+
+  public Duration getAiTranslationExpiryDuration() {
+    return aiTranslationExpiryDuration;
+  }
+
+  public void setAiTranslationExpiryDuration(Duration aiTranslationExpiryDuration) {
+    this.aiTranslationExpiryDuration = aiTranslationExpiryDuration;
+  }
 }

webapp/src/test/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartlingTest.java

@@ -34,6 +34,7 @@
 import com.box.l10n.mojito.quartz.QuartzJobInfo;
 import com.box.l10n.mojito.quartz.QuartzPollableTaskScheduler;
 import com.box.l10n.mojito.quartz.QuartzSchedulerManager;
+import com.box.l10n.mojito.service.ai.translation.AITranslationConfiguration;
 import com.box.l10n.mojito.service.asset.AssetService;
 import com.box.l10n.mojito.service.assetExtraction.AssetExtractionRepository;
 import com.box.l10n.mojito.service.assetExtraction.AssetExtractionService;
@@ -161,6 +162,8 @@ public class ThirdPartyTMSSmartlingTest extends ServiceTestBase {
 
   @Mock TextUnitBatchImporterService mockTextUnitBatchImporterService;
 
+  @Mock AITranslationConfiguration aiTranslationConfiguration;
+
   @Captor ArgumentCaptor<List<TextUnitDTO>> textUnitListCaptor;
 
   @Captor
@@ -216,7 +219,8 @@ public void setUp() throws SchedulerException {
             mockThirdPartyTMSSmartlingGlossary,
             assetTextUnitToTMTextUnitRepository,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
 
     mapper = new AndroidStringDocumentMapper(pluralSep, null);
     RetryBackoffSpec retryConfiguration =
@@ -292,7 +296,8 @@ public void testPushInBatchesWithSingularsAndNoPlurals()
             assetTextUnitToTMTextUnitRepository,
             batchSize,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
 
     TM tm = repository.getTm();
     Asset asset =
@@ -346,7 +351,8 @@ public void testRetryDuringPush() throws RepositoryNameAlreadyUsedException {
             assetTextUnitToTMTextUnitRepository,
             batchSize,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
     // throw timeout exception for first request, following request should be successful
     when(smartlingClient.uploadFile(any(), any(), any(), any(), any(), any(), any()))
         .thenThrow(
@@ -412,7 +418,8 @@ public void testRetriesExhaustedDuringPush() throws RepositoryNameAlreadyUsedExc
             assetTextUnitToTMTextUnitRepository,
             batchSize,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
 
     TM tm = repository.getTm();
     Asset asset =
@@ -463,7 +470,8 @@ public void testPushInBatchesWithNoSingularsAndPlurals()
             assetTextUnitToTMTextUnitRepository,
             batchSize,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
 
     TM tm = repository.getTm();
     Asset asset =
@@ -522,7 +530,8 @@ public void testPushInBatchesWithSingularsAndPlurals() throws RepositoryNameAlre
             assetTextUnitToTMTextUnitRepository,
             batchSize,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
 
     TM tm = repository.getTm();
     Asset asset =
@@ -737,7 +746,8 @@ public void testPullNoBatches() throws RepositoryLocaleCreationException, Interr
             mockThirdPartyTMSSmartlingGlossary,
             assetTextUnitToTMTextUnitRepository,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
     tmsSmartling.pull(
         repository,
         "projectId",
@@ -809,7 +819,8 @@ public void testPullNoBatchesPluralFix() throws RepositoryLocaleCreationExceptio
             mockThirdPartyTMSSmartlingGlossary,
             assetTextUnitToTMTextUnitRepository,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
     tmsSmartling.pull(
         repository,
         "projectId",
@@ -872,7 +883,8 @@ public void testPullDryRunNoBatches() throws RepositoryLocaleCreationException {
             mockThirdPartyTMSSmartlingGlossary,
             assetTextUnitToTMTextUnitRepository,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
     tmsSmartling.pull(
         repository,
         "projectId",
@@ -1461,7 +1473,8 @@ public void testPushTranslationsInBatches()
             assetTextUnitToTMTextUnitRepository,
             batchSize,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
     Repository repository =
         repositoryService.createRepository(testIdWatcher.getEntityName("batchRepo"));
     Locale frCA = localeService.findByBcp47Tag("fr-CA");
@@ -1615,7 +1628,8 @@ public void testBatchesFor() {
             assetTextUnitToTMTextUnitRepository,
             3,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
 
     assertThat(tmsSmartling.batchesFor(0)).isEqualTo(0);
     assertThat(tmsSmartling.batchesFor(1)).isEqualTo(1);
@@ -1638,7 +1652,8 @@ public void testBatchesFor() {
             assetTextUnitToTMTextUnitRepository,
             35,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
 
     assertThat(tmsSmartling.batchesFor(0)).isEqualTo(0);
     assertThat(tmsSmartling.batchesFor(1)).isEqualTo(1);
@@ -1659,7 +1674,8 @@ public void testBatchesFor() {
             assetTextUnitToTMTextUnitRepository,
             4231,
             meterRegistry,
-            mockQuartzPollableTaskScheduler);
+            mockQuartzPollableTaskScheduler,
+            aiTranslationConfiguration);
 
     assertThat(tmsSmartling.batchesFor(0)).isEqualTo(0);
     assertThat(tmsSmartling.batchesFor(1)).isEqualTo(1);