Skip to content

Commit

Permalink
AI translation import to Smartling via POST_TRANSLATION upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@maallen
maallen committed Oct 18, 2024
1 parent c18e41a commit 5ef5679
Show file tree
Hide file tree
Showing 5 changed files with 178 additions and 67 deletions.
2 changes: 1 addition & 1 deletion restclient/src/main/java/com/box/l10n/mojito/rest/ThirdPartySyncAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,5 +7,5 @@ public enum ThirdPartySyncAction {
PULL_SOURCE,
MAP_TEXTUNIT,
PUSH_SCREENSHOT,
PUSH_MT_TRANSLATION,
PUSH_AI_TRANSLATION,
}
206 changes: 147 additions & 59 deletions webapp/src/main/java/com/box/l10n/mojito/service/thirdparty/ThirdPartyTMSSmartling.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -761,101 +761,107 @@ public void pushAITranslations(
.tag("repository", repository.getName())) {
SmartlingOptions options = SmartlingOptions.parseList(optionList);
if (options.isJsonSync()) {
// TODO(mallen) Need to add json sync implementation
// TODO: Add json sync implementation
throw new RuntimeException("Not yet implemented");
}

List<SmartlingFile> result;

AndroidStringDocumentMapper mapper = new AndroidStringDocumentMapper(pluralSeparator, null);

Set<Long> filterTmTextUnitIds = getFilterTmTextUnitIdsForPushTranslation(options);

result =
List<SmartlingFile> result =
repository.getRepositoryLocales().stream()
.map(l -> l.getLocale().getBcp47Tag())
.filter(
localeTag ->
!localeTag.equalsIgnoreCase(repository.getSourceLocale().getBcp47Tag()))
.flatMap(
localeTag ->
Stream.concat(
mapWithIndex(
partitionSingulars(
repository.getId(),
localeTag,
skipTextUnitsWithPattern,
skipAssetsWithPathPattern,
includeTextUnitsWithPattern,
StatusFilter.MT_TRANSLATED),
(list, batch) ->
processAiTranslationBatch(
list,
batch,
localeTag,
mapper,
repository,
projectId,
options,
localeMapping,
Prefix.SINGULAR,
filterTmTextUnitIds)),
mapWithIndex(
partitionPlurals(
repository.getId(),
localeTag,
skipTextUnitsWithPattern,
skipAssetsWithPathPattern,
options.getPluralFixForLocales(),
includeTextUnitsWithPattern,
StatusFilter.MT_TRANSLATED),
(list, batch) ->
processAiTranslationBatch(
list,
batch,
localeTag,
mapper,
repository,
projectId,
options,
localeMapping,
Prefix.PLURAL,
filterTmTextUnitIds))))
localeTag -> {
Map<String, List<TextUnitDTO>> singularsByUploadedFileUri =
partitionSingulars(
repository.getId(),
localeTag,
skipTextUnitsWithPattern,
skipAssetsWithPathPattern,
includeTextUnitsWithPattern,
StatusFilter.MT_TRANSLATED)
.flatMap(List::stream)
.collect(Collectors.groupingBy(TextUnitDTO::getUploadedFileUri));

Map<String, List<TextUnitDTO>> pluralsByUploadedFileUri =
partitionPlurals(
repository.getId(),
localeTag,
skipTextUnitsWithPattern,
skipAssetsWithPathPattern,
options.getPluralFixForLocales(),
includeTextUnitsWithPattern,
StatusFilter.MT_TRANSLATED)
.flatMap(List::stream)
.collect(Collectors.groupingBy(TextUnitDTO::getUploadedFileUri));

Stream<SmartlingFile> singularFiles =
singularsByUploadedFileUri.entrySet().stream()
.map(
entry ->
uploadAiTranslationBatch(
entry.getValue(),
entry.getKey(),
localeTag,
mapper,
repository,
projectId,
options,
localeMapping,
filterTmTextUnitIds));

Stream<SmartlingFile> pluralFiles =
pluralsByUploadedFileUri.entrySet().stream()
.map(
entry ->
uploadAiTranslationBatch(
entry.getValue(),
entry.getKey(),
localeTag,
mapper,
repository,
projectId,
options,
localeMapping,
filterTmTextUnitIds));

return Stream.concat(singularFiles, pluralFiles);
})
.collect(Collectors.toList());

resultProcessor.processPushAiTranslations(result, options);
}
}

private SmartlingFile processAiTranslationBatch(
private SmartlingFile uploadAiTranslationBatch(
List<TextUnitDTO> batch,
Long batchNumber,
String uploadedFileUri,
String localeTag,
AndroidStringDocumentMapper mapper,
Repository repository,
String projectId,
SmartlingOptions options,
Map<String, String> localeMapping,
Prefix filePrefix,
Set<Long> filterTmTextUnitIds) {

SmartlingFile file =
processTranslationBatch(
uploadAiTranslationFile(
batch,
batchNumber,
localeTag,
uploadedFileUri,
mapper,
repository,
projectId,
options,
localeMapping,
filePrefix,
filterTmTextUnitIds,
POST_TRANSLATION);
// Update the status of the variants to MT review to ensure that we don't repeatedly upload the
// same translations on each sync until they're marked as APPROVED
filterTmTextUnitIds);
aiTranslationService.updateVariantStatusToMTReview(
batch.stream().map(TextUnitDTO::getTmTextUnitCurrentVariantId).toList());

return file;
}

Expand Down Expand Up @@ -948,6 +954,87 @@ private SmartlingFile processTranslationBatch(
}
}

private SmartlingFile uploadAiTranslationFile(
List<TextUnitDTO> batch,
String localeTag,
String fileName,
AndroidStringDocumentMapper mapper,
Repository repository,
String projectId,
SmartlingOptions options,
Map<String, String> localeMapping,
Set<Long> filterTmTextUnitIds) {

try (var timer =
Timer.resource(meterRegistry, "SmartlingSync.processTranslationBatch")
.tag("repository", repository.getName())
.tag("locale", localeTag)) {

logger.debug("Process translation batch for file: {}", fileName);
List<TextUnitDTO> fileBatch = batch;
SmartlingFile file = new SmartlingFile();
file.setFileName(fileName);

try {
logger.debug("Save target file to: {}", file.getFileName());

if (filterTmTextUnitIds != null) {
fileBatch =
fileBatch.stream()
.filter(
textUnitDTO -> filterTmTextUnitIds.contains(textUnitDTO.getTmTextUnitId()))
.collect(Collectors.toList());
}

AndroidStringDocumentWriter writer =
new AndroidStringDocumentWriter(mapper.readFromTargetTextUnits(fileBatch));
file.setFileContent(writer.toText());

} catch (ParserConfigurationException | TransformerException e) {
logger.error("An error occurred when processing a translation batch", e);
throw new RuntimeException(e);
}

if (!options.isDryRun()) {
logger.debug(
"Pushing Android file to Smartling project: {} and locale: {}", projectId, localeTag);
Mono.fromCallable(
() ->
smartlingClient.uploadLocalizedFile(
projectId,
fileName,
"android",
getSmartlingLocale(localeMapping, localeTag),
file.getFileContent(),
options.getPlaceholderFormat(),
options.getCustomPlaceholderFormat(),
POST_TRANSLATION))
.retryWhen(
smartlingClient
.getRetryConfiguration()
.doBeforeRetry(
e ->
logger.info(
String.format(
"Retrying after failure to upload localized file: %s, project id: %s",
fileName, projectId),
e.failure())))
.doOnError(
e -> {
String msg =
String.format(
"Error uploading localized file to Smartling for file %s in project %s",
fileName, projectId);
logger.error(msg, e);
throw new SmartlingClientException(msg, e);
})
.block();
}

return file;
}
}

private Stream<List<TextUnitDTO>> partitionSingulars(
Long repositoryId,
String localeTag,
Expand Down Expand Up @@ -1004,6 +1091,7 @@ private Stream<List<TextUnitDTO>> partitionSingulars(
null,
includeTextUnitWithPattern);
parameters.setOrderByTextUnitID(true);
parameters.setStatusFilter(statusFilter);
return partitionedStream(parameters, textUnitSearcher::search);
}

Expand Down Expand Up @@ -1095,7 +1183,7 @@ private Stream<List<TextUnitDTO>> partitionPlurals(
statusFilter);

parameters.setOrderByTextUnitID(true);

parameters.setStatusFilter(statusFilter);
return partitionedStream(parameters, searchFunction);
}

Expand Down
9 changes: 9 additions & 0 deletions webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public class TextUnitDTO {
private Long assetTextUnitId;
private ZonedDateTime tmTextUnitCreatedDate;
private boolean doNotTranslate;
private String uploadedFileUri;

public Long getTmTextUnitId() {
return tmTextUnitId;
Expand Down Expand Up @@ -242,4 +243,12 @@ public boolean isDoNotTranslate() {
public void setDoNotTranslate(boolean doNotTranslate) {
this.doNotTranslate = doNotTranslate;
}

public String getUploadedFileUri() {
return uploadedFileUri;
}

public void setUploadedFileUri(String uploadedFileUri) {
this.uploadedFileUri = uploadedFileUri;
}
}
4 changes: 4 additions & 0 deletions ...pp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitDTONativeObjectMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ public TextUnitDTO mapObject(CriteriaResult cr) {
String doNotTranslate = cr.getString(idx++);
t.setDoNotTranslate(Boolean.valueOf(doNotTranslate));

if (cr.hasProperty("uploadedFileUri")) {
t.setUploadedFileUri(cr.getString(idx++));
}

return t;
}

Expand Down
24 changes: 17 additions & 7 deletions webapp/src/main/java/com/box/l10n/mojito/service/tm/search/TextUnitSearcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,14 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters)
"tm_text_unit_pending_mt", "tmtupmt", "tmtupmt.tm_text_unit_id", "tu.id"));
}

if (searchParameters.getStatusFilter() != null
&& searchParameters.getStatusFilter().equals(StatusFilter.MT_TRANSLATED)) {
// Pushing AI translations, need to retrieve the uploadedFileUri from the ThirdPartyTextUnit
// table
c.addJoin(
NativeExps.innerJoin("third_party_text_unit", "tptu", "tptu.tm_text_unit_id", "tu.id"));
}

NativeJunctionExp onClauseRepositoryLocale = NativeExps.conjunction();
onClauseRepositoryLocale.add(new NativeColumnEqExp("rl.locale_id", "l.id"));
onClauseRepositoryLocale.add(new NativeColumnEqExp("rl.repository_id", "r.id"));
Expand Down Expand Up @@ -226,15 +234,11 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters)
"plural_form_for_locale", "pffl", NativeJoin.JoinType.LEFT_OUTER, onClausePluralForm));

logger.debug("Set projections");

// TODO(P1) Might want to some of those projection as optional for perf reason
c.setProjection(
NativeProjection projection =
NativeExps.projection()
.addProjection("tu.id", "tmTextUnitId")
.addProjection("tuv.id", "tmTextUnitVariantId")
.
// TODO(PO) THIS NOT CONSISTANT !! chooose
addProjection("l.id", "localeId")
.addProjection("l.id", "localeId")
.addProjection("l.bcp47_tag", "targetLocale")
.addProjection("tu.name", "name")
.addProjection("tu.content", "source")
Expand All @@ -256,7 +260,13 @@ NativeCriteria getCriteriaForSearch(TextUnitSearcherParameters searchParameters)
.addProjection("a.path", "assetPath")
.addProjection("atu.id", "assetTextUnitId")
.addProjection("tu.created_date", "tmTextUnitCreatedDate")
.addProjection("atu.do_not_translate", "doNotTranslate"));
.addProjection("atu.do_not_translate", "doNotTranslate");

if (searchParameters.getStatusFilter() != null
&& searchParameters.getStatusFilter().equals(StatusFilter.MT_TRANSLATED)) {
projection.addProjection("tptu.uploaded_file_uri", "uploadedFileUri");
}
c.setProjection(projection);

logger.debug("Add search filters");
NativeJunctionExp conjunction = NativeExps.conjunction();
Expand Down

0 comments on commit 5ef5679

Please sign in to comment.