diff --git a/pass-deposit-services/README.md b/pass-deposit-services/README.md index 4e47e62f3..584f42a9c 100644 --- a/pass-deposit-services/README.md +++ b/pass-deposit-services/README.md @@ -163,7 +163,7 @@ any messages relating to the resource. Failed `Deposit` resources will be retrie A resource will be considered as failed when errors occur during the processing of `Submission` and `Deposit` resources. Some errors may be caused by transient network issues, or a server being rebooted. In the case of such failures, Deposit Services will retry for n number of days after the `Submission` is created. The number of days -is set in an application property named `pass.deposit.update.window.days`. +is set in an application property named `pass.status.update.window.days`. `Submission` resources are failed when: @@ -187,7 +187,7 @@ the user interface, and resubmit it. See `DepositTask` for details. Deposits fail for transient reasons; a server being down, an interruption in network communication, or invalid credentials for the downstream repository are just a few examples. As stated, DS will retry failed `Deposit` resources for n number of days after the creation of the associated `Submission`. The number of days -is set in an application property named `pass.deposit.update.window.days`. +is set in an application property named `pass.status.update.window.days`. ## Build and Deployment diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositUpdater.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositUpdater.java index 344d9c86a..8f4de342f 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositUpdater.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositUpdater.java @@ -18,12 +18,12 @@ import java.io.IOException; import java.time.ZoneOffset; import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Objects; import org.eclipse.pass.deposit.config.repository.Repositories; import org.eclipse.pass.deposit.config.repository.RepositoryConfig; +import org.eclipse.pass.support.client.ModelUtil; import org.eclipse.pass.support.client.PassClient; import org.eclipse.pass.support.client.PassClientSelector; import org.eclipse.pass.support.client.RSQL; @@ -37,10 +37,7 @@ @Component public class DepositUpdater { - private static final Logger LOG = LoggerFactory.getLogger(DepositUpdater.class); - private static final DateTimeFormatter DATE_TIME_FORMATTER = - DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX"); private final PassClient passClient; private final DepositTaskHelper depositHelper; @@ -49,7 +46,7 @@ public class DepositUpdater { private final List repoKeysWithDepositProcessors; - @Value("${pass.deposit.update.window.days}") + @Value("${pass.status.update.window.days}") private long updateWindowDays; @Autowired @@ -73,7 +70,7 @@ private void retryFailedDeposits(ZonedDateTime submissionFromDate) throws IOExce failedDepositsSelector.setFilter( RSQL.and( RSQL.equals("depositStatus", DepositStatus.FAILED.getValue()), - RSQL.gte("submission.submittedDate", DATE_TIME_FORMATTER.format(submissionFromDate)) + RSQL.gte("submission.submittedDate", ModelUtil.dateTimeFormatter().format(submissionFromDate)) ) ); List failedDeposits = passClient.streamObjects(failedDepositsSelector).toList(); @@ -96,7 +93,7 @@ private void updateSubmittedDeposits(ZonedDateTime submissionFromDate) throws IO submittedDepositsSelector.setFilter( RSQL.and( RSQL.equals("depositStatus", DepositStatus.SUBMITTED.getValue()), - RSQL.gte("submission.submittedDate", DATE_TIME_FORMATTER.format(submissionFromDate)), + RSQL.gte("submission.submittedDate", ModelUtil.dateTimeFormatter().format(submissionFromDate)), RSQL.in("repository.repositoryKey", repoKeysWithDepositProcessors.toArray(new String[0])) ) ); diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/SubmissionStatusUpdater.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/SubmissionStatusUpdater.java index ddf848c30..d0d9a0a2e 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/SubmissionStatusUpdater.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/SubmissionStatusUpdater.java @@ -16,9 +16,11 @@ package org.eclipse.pass.deposit.service; import java.io.IOException; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.List; -import java.util.stream.Stream; +import org.eclipse.pass.support.client.ModelUtil; import org.eclipse.pass.support.client.PassClient; import org.eclipse.pass.support.client.PassClientSelector; import org.eclipse.pass.support.client.RSQL; @@ -28,6 +30,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; /** @@ -52,6 +55,9 @@ public class SubmissionStatusUpdater { private final SubmissionStatusService statusService; private final PassClient passClient; + @Value("${pass.status.update.window.days}") + private long updateWindowDays; + @Autowired public SubmissionStatusUpdater(SubmissionStatusService statusService, PassClient passClient) { this.statusService = statusService; @@ -63,11 +69,15 @@ public SubmissionStatusUpdater(SubmissionStatusService statusService, PassClient * @throws IOException io exception */ public void doUpdate() throws IOException { + ZonedDateTime submissionFromDate = ZonedDateTime.now(ZoneOffset.UTC).minusDays(updateWindowDays); + PassClientSelector sel = new PassClientSelector<>(Submission.class); sel.setFilter( RSQL.and( - RSQL.in("submissionStatus", getSubmissionStatusFilter()), - RSQL.equals("submitted", "true") + RSQL.out("submissionStatus", SubmissionStatus.COMPLETE.getValue(), + SubmissionStatus.CANCELLED.getValue()), + RSQL.equals("submitted", "true"), + RSQL.gte("submittedDate", ModelUtil.dateTimeFormatter().format(submissionFromDate)) ) ); List submissions = passClient.streamObjects(sel).toList(); @@ -88,12 +98,4 @@ public void doUpdate() throws IOException { } }); } - - private String[] getSubmissionStatusFilter() { - return Stream.of(SubmissionStatus.values()) - .filter(status -> status != SubmissionStatus.COMPLETE) - .filter(status -> status != SubmissionStatus.CANCELLED) - .map(SubmissionStatus::getValue).toArray(String[]::new); - } - } diff --git a/pass-deposit-services/deposit-core/src/main/resources/application.properties b/pass-deposit-services/deposit-core/src/main/resources/application.properties index af529ab7d..db8914399 100644 --- a/pass-deposit-services/deposit-core/src/main/resources/application.properties +++ b/pass-deposit-services/deposit-core/src/main/resources/application.properties @@ -41,7 +41,7 @@ pass.deposit.queue.submission.name=${PASS_DEPOSIT_QUEUE_SUBMISSION_NAME:submissi pass.deposit.transport.swordv2.sleep-time-ms=10000 pass.deposit.transport.swordv2.followRedirects=false -pass.deposit.update.window.days=10 +pass.status.update.window.days=10 pass.deposit.jobs.disabled=false # By default run all jobs every 10 minutes @@ -59,4 +59,4 @@ nihms.mail.password=${NIHMS_MAIL_PASSWORD} pass.deposit.nihms.email.enabled=false pass.deposit.nihms.email.delay=720000 -pass.deposit.nihms.email.from=${PASS_DEPOSIT_NIHMS_EMAIL_FROM:nihms-help@ncbi.nlm.nih.gov} \ No newline at end of file +pass.deposit.nihms.email.from=${PASS_DEPOSIT_NIHMS_EMAIL_FROM:nihms-help@ncbi.nlm.nih.gov} diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/AbstractDepositSubmissionIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/AbstractDepositSubmissionIT.java index 35906e47e..c6b12e221 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/AbstractDepositSubmissionIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/AbstractDepositSubmissionIT.java @@ -20,6 +20,7 @@ import java.io.FileReader; import java.io.IOException; import java.io.InputStream; +import java.time.ZonedDateTime; import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -138,6 +139,8 @@ public List createSubmission(InputStream submissionGraph) throws IOE public void triggerSubmission(Submission submission) throws IOException { submission.setSubmitted(true); submission.setSubmissionStatus(SubmissionStatus.SUBMITTED); + submission.setSubmittedDate(ZonedDateTime.now()); + passClient.updateObject(submission); } diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/service/SubmissionStatusUpdaterIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/service/SubmissionStatusUpdaterIT.java index 68bfe626a..25a0432ac 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/service/SubmissionStatusUpdaterIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/service/SubmissionStatusUpdaterIT.java @@ -23,6 +23,7 @@ import static org.mockito.Mockito.when; import java.io.IOException; +import java.time.ZonedDateTime; import java.util.List; import org.eclipse.pass.support.client.PassClientSelector; @@ -35,6 +36,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.mock.mockito.MockBean; /** @@ -45,6 +47,9 @@ public class SubmissionStatusUpdaterIT extends AbstractSubmissionIT { @Autowired private SubmissionStatusUpdater submissionStatusUpdater; @MockBean private SubmissionStatusService statusService; + @Value("${pass.status.update.window.days}") + private long updateWindowDays; + private Submission submission; @BeforeEach @@ -70,6 +75,7 @@ void testDoUpdate_Success() throws IOException { // GIVEN submission.setSubmissionStatus(SubmissionStatus.SUBMITTED); submission.setSubmitted(true); + submission.setSubmittedDate(ZonedDateTime.now()); passClient.updateObject(submission); when(statusService.calculateSubmissionStatus(any(Submission.class))).thenReturn(SubmissionStatus.COMPLETE); Mockito.clearInvocations(passClient); @@ -91,6 +97,7 @@ void testDoUpdate_Success_NoUpdateStatusNotChanged() throws IOException { // GIVEN submission.setSubmissionStatus(SubmissionStatus.SUBMITTED); submission.setSubmitted(true); + submission.setSubmittedDate(ZonedDateTime.now()); passClient.updateObject(submission); when(statusService.calculateSubmissionStatus(any(Submission.class))).thenReturn(SubmissionStatus.SUBMITTED); Mockito.clearInvocations(passClient); @@ -107,6 +114,7 @@ void testDoUpdate_Success_NoUpdateStatusIsNull() throws IOException { // GIVEN submission.setSubmissionStatus(null); submission.setSubmitted(true); + submission.setSubmittedDate(ZonedDateTime.now()); passClient.updateObject(submission); when(statusService.calculateSubmissionStatus(any(Submission.class))).thenReturn(SubmissionStatus.SUBMITTED); Mockito.clearInvocations(passClient); @@ -123,6 +131,23 @@ void testDoUpdate_Success_NoUpdateNotSubmitted() throws IOException { // GIVEN submission.setSubmissionStatus(SubmissionStatus.SUBMITTED); submission.setSubmitted(false); + submission.setSubmittedDate(ZonedDateTime.now()); + passClient.updateObject(submission); + Mockito.clearInvocations(passClient); + + // WHEN + submissionStatusUpdater.doUpdate(); + + // THEN + verify(passClient, times(0)).updateObject(any()); + } + + @Test + void testDoUpdate_Success_OutSideDateWindow() throws IOException { + // GIVEN + submission.setSubmissionStatus(SubmissionStatus.SUBMITTED); + submission.setSubmitted(true); + submission.setSubmittedDate(ZonedDateTime.now().minusDays(updateWindowDays + 1)); passClient.updateObject(submission); Mockito.clearInvocations(passClient);