Submission status job filters by submission date

pass-deposit-services/README.md

@@ -163,7 +163,7 @@ any messages relating to the resource. Failed `Deposit` resources will be retrie
 A resource will be considered as failed when errors occur during the processing of `Submission` and `Deposit` resources.
 Some errors may be caused by transient network issues, or a server being rebooted.  In the case of such failures,
 Deposit Services will retry for n number of days after the `Submission` is created. The number of days
-is set in an application property named `pass.deposit.update.window.days`.
+is set in an application property named `pass.status.update.window.days`.
 
 `Submission` resources are failed when:
 
@@ -187,7 +187,7 @@ the user interface, and resubmit it.
 See `DepositTask` for details. Deposits fail for transient reasons; a server being down, an interruption in network
 communication, or invalid credentials for the downstream repository are just a few examples. As stated, DS will retry
 failed `Deposit` resources for n number of days after the creation of the associated `Submission`.  The number of days
-is set in an application property named `pass.deposit.update.window.days`.
+is set in an application property named `pass.status.update.window.days`.
 
 ## Build and Deployment
 

pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/DepositUpdater.java

@@ -18,12 +18,12 @@
 import java.io.IOException;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
 import java.util.List;
 import java.util.Objects;
 
 import org.eclipse.pass.deposit.config.repository.Repositories;
 import org.eclipse.pass.deposit.config.repository.RepositoryConfig;
+import org.eclipse.pass.support.client.ModelUtil;
 import org.eclipse.pass.support.client.PassClient;
 import org.eclipse.pass.support.client.PassClientSelector;
 import org.eclipse.pass.support.client.RSQL;
@@ -37,10 +37,7 @@
 
 @Component
 public class DepositUpdater {
-
     private static final Logger LOG = LoggerFactory.getLogger(DepositUpdater.class);
-    private static final DateTimeFormatter DATE_TIME_FORMATTER =
-        DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX");
 
     private final PassClient passClient;
     private final DepositTaskHelper depositHelper;
@@ -49,7 +46,7 @@ public class DepositUpdater {
 
     private final List<String> repoKeysWithDepositProcessors;
 
-    @Value("${pass.deposit.update.window.days}")
+    @Value("${pass.status.update.window.days}")
     private long updateWindowDays;
 
     @Autowired
@@ -73,7 +70,7 @@ private void retryFailedDeposits(ZonedDateTime submissionFromDate) throws IOExce
         failedDepositsSelector.setFilter(
             RSQL.and(
                 RSQL.equals("depositStatus", DepositStatus.FAILED.getValue()),
-                RSQL.gte("submission.submittedDate", DATE_TIME_FORMATTER.format(submissionFromDate))
+                RSQL.gte("submission.submittedDate", ModelUtil.dateTimeFormatter().format(submissionFromDate))
             )
         );
         List<Deposit> failedDeposits = passClient.streamObjects(failedDepositsSelector).toList();
@@ -96,7 +93,7 @@ private void updateSubmittedDeposits(ZonedDateTime submissionFromDate) throws IO
         submittedDepositsSelector.setFilter(
             RSQL.and(
                 RSQL.equals("depositStatus", DepositStatus.SUBMITTED.getValue()),
-                RSQL.gte("submission.submittedDate", DATE_TIME_FORMATTER.format(submissionFromDate)),
+                RSQL.gte("submission.submittedDate", ModelUtil.dateTimeFormatter().format(submissionFromDate)),
                 RSQL.in("repository.repositoryKey", repoKeysWithDepositProcessors.toArray(new String[0]))
             )
         );

pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/service/SubmissionStatusUpdater.java

@@ -16,9 +16,11 @@
 package org.eclipse.pass.deposit.service;
 
 import java.io.IOException;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
 import java.util.List;
-import java.util.stream.Stream;
 
+import org.eclipse.pass.support.client.ModelUtil;
 import org.eclipse.pass.support.client.PassClient;
 import org.eclipse.pass.support.client.PassClientSelector;
 import org.eclipse.pass.support.client.RSQL;
@@ -28,6 +30,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 /**
@@ -52,6 +55,9 @@ public class SubmissionStatusUpdater {
     private final SubmissionStatusService statusService;
     private final PassClient passClient;
 
+    @Value("${pass.status.update.window.days}")
+    private long updateWindowDays;
+
     @Autowired
     public SubmissionStatusUpdater(SubmissionStatusService statusService, PassClient passClient) {
         this.statusService = statusService;
@@ -63,11 +69,15 @@ public SubmissionStatusUpdater(SubmissionStatusService statusService, PassClient
      * @throws IOException io exception
      */
     public void doUpdate() throws IOException {
+        ZonedDateTime submissionFromDate = ZonedDateTime.now(ZoneOffset.UTC).minusDays(updateWindowDays);
+
         PassClientSelector<Submission> sel = new PassClientSelector<>(Submission.class);
         sel.setFilter(
             RSQL.and(
-                RSQL.in("submissionStatus", getSubmissionStatusFilter()),
-                RSQL.equals("submitted", "true")
+                RSQL.out("submissionStatus", SubmissionStatus.COMPLETE.getValue(),
+                        SubmissionStatus.CANCELLED.getValue()),
+                RSQL.equals("submitted", "true"),
+                RSQL.gte("submittedDate", ModelUtil.dateTimeFormatter().format(submissionFromDate))
             )
         );
         List<Submission> submissions = passClient.streamObjects(sel).toList();
@@ -88,12 +98,4 @@ public void doUpdate() throws IOException {
             }
         });
     }
-
-    private String[] getSubmissionStatusFilter() {
-        return Stream.of(SubmissionStatus.values())
-            .filter(status -> status != SubmissionStatus.COMPLETE)
-            .filter(status -> status != SubmissionStatus.CANCELLED)
-            .map(SubmissionStatus::getValue).toArray(String[]::new);
-    }
-
 }

pass-deposit-services/deposit-core/src/main/resources/application.properties

@@ -41,7 +41,7 @@ pass.deposit.queue.submission.name=${PASS_DEPOSIT_QUEUE_SUBMISSION_NAME:submissi
 pass.deposit.transport.swordv2.sleep-time-ms=10000
 pass.deposit.transport.swordv2.followRedirects=false
 
-pass.deposit.update.window.days=10
+pass.status.update.window.days=10
 
 pass.deposit.jobs.disabled=false
 # By default run all jobs every 10 minutes
@@ -59,4 +59,4 @@ nihms.mail.password=${NIHMS_MAIL_PASSWORD}
 
 pass.deposit.nihms.email.enabled=false
 pass.deposit.nihms.email.delay=720000
-pass.deposit.nihms.email.from=${PASS_DEPOSIT_NIHMS_EMAIL_FROM:[email protected]}
+pass.deposit.nihms.email.from=${PASS_DEPOSIT_NIHMS_EMAIL_FROM:[email protected]}

pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/AbstractDepositSubmissionIT.java

@@ -20,6 +20,7 @@
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.InputStream;
+import java.time.ZonedDateTime;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
@@ -138,6 +139,8 @@ public List<PassEntity> createSubmission(InputStream submissionGraph) throws IOE
     public void triggerSubmission(Submission submission) throws IOException {
         submission.setSubmitted(true);
         submission.setSubmissionStatus(SubmissionStatus.SUBMITTED);
+        submission.setSubmittedDate(ZonedDateTime.now());
+
         passClient.updateObject(submission);
     }
 

pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/service/SubmissionStatusUpdaterIT.java

@@ -23,6 +23,7 @@
 import static org.mockito.Mockito.when;
 
 import java.io.IOException;
+import java.time.ZonedDateTime;
 import java.util.List;
 
 import org.eclipse.pass.support.client.PassClientSelector;
@@ -70,6 +71,7 @@ void testDoUpdate_Success() throws IOException {
         // GIVEN
         submission.setSubmissionStatus(SubmissionStatus.SUBMITTED);
         submission.setSubmitted(true);
+        submission.setSubmittedDate(ZonedDateTime.now());
         passClient.updateObject(submission);
         when(statusService.calculateSubmissionStatus(any(Submission.class))).thenReturn(SubmissionStatus.COMPLETE);
         Mockito.clearInvocations(passClient);
@@ -91,6 +93,7 @@ void testDoUpdate_Success_NoUpdateStatusNotChanged() throws IOException {
         // GIVEN
         submission.setSubmissionStatus(SubmissionStatus.SUBMITTED);
         submission.setSubmitted(true);
+        submission.setSubmittedDate(ZonedDateTime.now());
         passClient.updateObject(submission);
         when(statusService.calculateSubmissionStatus(any(Submission.class))).thenReturn(SubmissionStatus.SUBMITTED);
         Mockito.clearInvocations(passClient);
@@ -107,6 +110,7 @@ void testDoUpdate_Success_NoUpdateStatusIsNull() throws IOException {
         // GIVEN
         submission.setSubmissionStatus(null);
         submission.setSubmitted(true);
+        submission.setSubmittedDate(ZonedDateTime.now());
         passClient.updateObject(submission);
         when(statusService.calculateSubmissionStatus(any(Submission.class))).thenReturn(SubmissionStatus.SUBMITTED);
         Mockito.clearInvocations(passClient);
@@ -123,6 +127,7 @@ void testDoUpdate_Success_NoUpdateNotSubmitted() throws IOException {
         // GIVEN
         submission.setSubmissionStatus(SubmissionStatus.SUBMITTED);
         submission.setSubmitted(false);
+        submission.setSubmittedDate(ZonedDateTime.now());
         passClient.updateObject(submission);
         Mockito.clearInvocations(passClient);