GRAD2-2920 Scheduled purge of old events and event history (#343)

* Added cascade deletion of old events including history.

* Added cron schedule every day at midnight

* Added more test coverage.

* Added more test coverage.

* Updated exception to be thrown from repository layer.

---------

Co-authored-by: chris.ditcher <[email protected]>

api/src/main/java/ca/bc/gov/educ/api/trax/model/entity/EventEntity.java

@@ -88,6 +88,9 @@ public class EventEntity {
  @Column(name = "ACTIVITY_CODE")
  private String activityCode;
 
+ @OneToOne(cascade = CascadeType.ALL, mappedBy = "event")
+ private EventHistoryEntity eventHistoryEntity;
+
  /**
  * Gets event payload.
  *

api/src/main/java/ca/bc/gov/educ/api/trax/repository/EventHistoryRepository.java

@@ -7,6 +7,4 @@
 
 @Repository
 public interface EventHistoryRepository extends JpaRepository<EventHistoryEntity, UUID> {
-
-
 }

api/src/main/java/ca/bc/gov/educ/api/trax/repository/EventRepository.java

@@ -2,8 +2,6 @@
 
 import ca.bc.gov.educ.api.trax.model.entity.EventEntity;
 import org.springframework.data.jpa.repository.JpaRepository;
-import org.springframework.data.jpa.repository.Modifying;
-import org.springframework.data.jpa.repository.Query;
 import org.springframework.stereotype.Repository;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -20,8 +18,6 @@ public interface EventRepository extends JpaRepository<EventEntity, UUID> {
  List<EventEntity> findAllByEventStatusOrderByCreateDate(String eventStatus);
 
  @Transactional
- @Modifying
- @Query("delete from EventEntity where createDate <= :createDate")
- void deleteByCreateDateBefore(LocalDateTime createDate);
+ void deleteByCreateDateLessThan(LocalDateTime createDate);
 
 }

api/src/main/java/ca/bc/gov/educ/api/trax/scheduler/PurgeOldRecordsScheduler.java

@@ -1,12 +1,13 @@
 package ca.bc.gov.educ.api.trax.scheduler;
 
-import ca.bc.gov.educ.api.trax.repository.EventRepository;
 import ca.bc.gov.educ.api.trax.repository.TraxUpdatedPubEventRepository;
+import ca.bc.gov.educ.api.trax.service.EventHistoryService;
 import ca.bc.gov.educ.api.trax.util.EducGradTraxApiConstants;
 import jakarta.transaction.Transactional;
 import lombok.extern.slf4j.Slf4j;
 import net.javacrumbs.shedlock.core.LockAssert;
 import net.javacrumbs.shedlock.spring.annotation.SchedulerLock;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
@@ -16,14 +17,13 @@
 @Slf4j
 public class PurgeOldRecordsScheduler {
 
- private final EventRepository eventRepository;
+ private final EventHistoryService eventHistoryService;
  private final TraxUpdatedPubEventRepository traxUpdatedPubEventRepository;
  private final EducGradTraxApiConstants constants;
 
- public PurgeOldRecordsScheduler(final EventRepository eventRepository,
- final TraxUpdatedPubEventRepository traxUpdatedPubEventRepository,
- final EducGradTraxApiConstants constants) {
- this.eventRepository = eventRepository;
+ @Autowired
+ public PurgeOldRecordsScheduler(EventHistoryService eventHistoryService, TraxUpdatedPubEventRepository traxUpdatedPubEventRepository, EducGradTraxApiConstants constants) {
+ this.eventHistoryService = eventHistoryService;
  this.traxUpdatedPubEventRepository = traxUpdatedPubEventRepository;
  this.constants = constants;
  }
@@ -33,10 +33,15 @@ public PurgeOldRecordsScheduler(final EventRepository eventRepository,
  lockAtLeastFor = "PT1H", lockAtMostFor = "PT1H") //midnight job so lock for an hour
  @Transactional
  public void purgeOldRecords() {
- LockAssert.assertLocked();
- final LocalDateTime createDateToCompare = this.calculateCreateDateBasedOnStaleEventInDays();
- this.eventRepository.deleteByCreateDateBefore(createDateToCompare);
- this.traxUpdatedPubEventRepository.deleteByCreateDateBefore(createDateToCompare);
+ try {
+ LockAssert.assertLocked();
+ final LocalDateTime createDateToCompare = this.calculateCreateDateBasedOnStaleEventInDays();
+ this.eventHistoryService.purgeOldEventAndEventHistoryRecords(createDateToCompare);
+ this.traxUpdatedPubEventRepository.deleteByCreateDateBefore(createDateToCompare);
+ } catch (Exception e) {
+ log.error(e.getMessage());
+ }
+
  }
 
  private LocalDateTime calculateCreateDateBasedOnStaleEventInDays() {

api/src/main/java/ca/bc/gov/educ/api/trax/service/EventHistoryService.java

@@ -0,0 +1,31 @@
+package ca.bc.gov.educ.api.trax.service;
+
+import ca.bc.gov.educ.api.trax.exception.ServiceException;
+import ca.bc.gov.educ.api.trax.repository.EventRepository;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.time.LocalDateTime;
+
+@Slf4j
+@Service
+public class EventHistoryService {
+
+ private EventRepository eventRepository;
+
+ @Autowired
+ public EventHistoryService(EventRepository eventRepository) {
+ this.eventRepository = eventRepository;
+ }
+
+ public void purgeOldEventAndEventHistoryRecords(LocalDateTime sinceBefore) throws ServiceException {
+ try {
+ this.eventRepository.deleteByCreateDateLessThan(sinceBefore);
+ } catch (Exception e) {
+ throw new ServiceException(String.format("Exception encountered when attempting old Event History Purge: %s", e.getMessage()));
+ }
+ }
+
+
+}

api/src/test/java/ca/bc/gov/educ/api/trax/service/EventHistoryServiceMockTest.java

@@ -0,0 +1,32 @@
+package ca.bc.gov.educ.api.trax.service;
+
+import ca.bc.gov.educ.api.trax.exception.ServiceException;
+import ca.bc.gov.educ.api.trax.repository.EventRepository;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.mock.mockito.MockBean;
+
+import java.time.LocalDateTime;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.doThrow;
+
+
+class EventHistoryServiceMockTest extends BaseReplicationServiceTest {
+
+ @Autowired
+ EventHistoryService eventHistoryService;
+
+ @MockBean
+ EventRepository eventRepository;
+
+ @Test
+ void purgeOldEventAndEventHistoryRecords_givenExceptionThrown_shouldThrowException() {
+ final String ERROR_MSG = "Exception encountered";
+ final LocalDateTime localDateTime = LocalDateTime.now();
+ doThrow(new RuntimeException(ERROR_MSG)).when(eventRepository).deleteByCreateDateLessThan(localDateTime);
+ Exception exception = assertThrows(ServiceException.class, () -> eventHistoryService.purgeOldEventAndEventHistoryRecords(localDateTime));
+ assertTrue(exception.getMessage().contains(ERROR_MSG));
+ }
+}

api/src/test/java/ca/bc/gov/educ/api/trax/service/EventHistoryServiceTest.java

@@ -0,0 +1,64 @@
+package ca.bc.gov.educ.api.trax.service;
+
+import ca.bc.gov.educ.api.trax.model.entity.EventEntity;
+import ca.bc.gov.educ.api.trax.model.entity.EventHistoryEntity;
+import ca.bc.gov.educ.api.trax.repository.EventHistoryRepository;
+import ca.bc.gov.educ.api.trax.repository.EventRepository;
+import ca.bc.gov.educ.api.trax.support.TestUtils;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.junit.Assert;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.time.LocalDateTime;
+import java.util.Optional;
+
+public class EventHistoryServiceTest extends BaseReplicationServiceTest {
+ @Autowired
+ private EventHistoryService eventHistoryService;
+ final LocalDateTime purgeTimeInDays = LocalDateTime.now().minusDays(2);
+
+ @Test
+ public void testDeleteEvent_giventEventHistory_ShouldCascadeDelete() throws JsonProcessingException {
+ // set up
+ EventRepository eventRepository = this.replicationTestUtils.getEventRepository();
+ EventHistoryRepository eventHistoryRepository = this.replicationTestUtils.getEventHistoryRepository();
+ var event = TestUtils.createEvent("DELETE_DISTRICT_CONTACT", TestUtils.createDistrictContact(), LocalDateTime.now(), eventRepository);
+ var eventHistory = TestUtils.createEventHistory(event, LocalDateTime.now(), eventHistoryRepository);
+ eventRepository.deleteById(event.getReplicationEventId());
+ Optional<EventEntity> eventThatShouldBePurgedOptional = eventRepository.findById(event.getReplicationEventId());
+ Optional<EventHistoryEntity> eventHistoryThatShouldBePurgedAlso = eventHistoryRepository.findById(eventHistory.getId());
+ Assert.assertTrue(eventHistoryThatShouldBePurgedAlso.isEmpty() && eventThatShouldBePurgedOptional.isEmpty());
+ }
+
+ @Test
+ public void purgeOldEventAndEventHistoryRecords_givenNoExceptionAndOldRecord_shouldPurgeRecords() throws JsonProcessingException {
+ // set up
+ EventRepository eventRepository = this.replicationTestUtils.getEventRepository();
+ EventHistoryRepository eventHistoryRepository = this.replicationTestUtils.getEventHistoryRepository();
+ var eventAge = LocalDateTime.now().minusDays(3);
+ var eventThatShouldBePurged = TestUtils.createEvent("DELETE_DISTRICT_CONTACT", TestUtils.createDistrictContact(), eventAge, eventRepository);
+ // set up event history for eventThatShouldBePurged
+ var eventHistory = TestUtils.createEventHistory(eventThatShouldBePurged, eventAge, eventHistoryRepository);
+ // call purge
+ eventHistoryService.purgeOldEventAndEventHistoryRecords(purgeTimeInDays);
+ // check repo and ensure that older record purged
+ Optional<EventEntity> eventThatShouldBePurgedOptional = eventRepository.findById(eventThatShouldBePurged.getReplicationEventId());
+ Optional<EventHistoryEntity> eventHistoryThatShouldBePurgedAlso = eventHistoryRepository.findById(eventHistory.getId());
+ Assert.assertTrue(eventHistoryThatShouldBePurgedAlso.isEmpty() && eventThatShouldBePurgedOptional.isEmpty());
+ }
+
+ @Test
+ public void purgeOldEventAndEventHistoryRecords_givenNoExceptionAndNewRecord_shouldNotPurgeRecords() throws JsonProcessingException {
+ EventRepository eventRepository = this.replicationTestUtils.getEventRepository();
+ EventHistoryRepository eventHistoryRepository = this.replicationTestUtils.getEventHistoryRepository();
+ var eventThatShouldNotBePurged = TestUtils.createEvent("DELETE_DISTRICT_CONTACT", TestUtils.createDistrictContact(), eventRepository);
+ var eventHistory = TestUtils.createEventHistory(eventThatShouldNotBePurged, LocalDateTime.now(), eventHistoryRepository);
+ // call purge
+ eventHistoryService.purgeOldEventAndEventHistoryRecords(purgeTimeInDays);
+ // check repo and ensure that new record is not purged
+ Optional<EventEntity> eventThatShouldNotBePurgedOptional = eventRepository.findById(eventThatShouldNotBePurged.getReplicationEventId());
+ Optional<EventHistoryEntity> eventHistoryThatShouldNotBePurgedAlso = eventHistoryRepository.findById(eventHistory.getId());
+ Assert.assertTrue(eventHistoryThatShouldNotBePurgedAlso.isPresent() && eventThatShouldNotBePurgedOptional.isPresent());
+ }
+}

api/src/test/java/ca/bc/gov/educ/api/trax/support/TestUtils.java

@@ -6,7 +6,9 @@
 import ca.bc.gov.educ.api.trax.model.dto.SchoolContact;
 import ca.bc.gov.educ.api.trax.model.dto.institute.*;
 import ca.bc.gov.educ.api.trax.model.entity.EventEntity;
+import ca.bc.gov.educ.api.trax.model.entity.EventHistoryEntity;
 import ca.bc.gov.educ.api.trax.model.entity.TraxStudentEntity;
+import ca.bc.gov.educ.api.trax.repository.EventHistoryRepository;
 import ca.bc.gov.educ.api.trax.repository.EventRepository;
 import ca.bc.gov.educ.api.trax.util.JsonUtil;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -39,6 +41,10 @@ public static GradStatusEventPayloadDTO createGraduationStatus(boolean isGraduat
  }
 
  public static EventEntity createEvent(String eventType, Object payload, EventRepository eventRepository) throws JsonProcessingException {
+ return createEvent(eventType, payload, LocalDateTime.now(), eventRepository);
+ }
+
+ public static EventEntity createEvent(String eventType, Object payload, LocalDateTime createDate, EventRepository eventRepository) throws JsonProcessingException {
  var event = EventEntity.builder()
  .eventType(eventType)
  .eventId(UUID.randomUUID())
@@ -47,13 +53,25 @@ public static EventEntity createEvent(String eventType, Object payload, EventRep
  .eventStatus(DB_COMMITTED.toString())
  .createUser(DEFAULT_CREATED_BY)
  .updateUser(DEFAULT_UPDATED_BY)
- .createDate(LocalDateTime.now())
+ .createDate(createDate)
  .updateDate(LocalDateTime.now())
  .build();
  eventRepository.save(event);
  return event;
  }
 
+ public static EventHistoryEntity createEventHistory(EventEntity event, LocalDateTime createdDate, EventHistoryRepository eventHistoryRepository) {
+ var eventHistory = new EventHistoryEntity();
+ eventHistory.setEvent(event);
+ eventHistory.setAcknowledgeFlag("N");
+ eventHistory.setCreateDate(createdDate);
+ eventHistory.setCreateUser("TEST");
+ eventHistory.setUpdateDate(LocalDateTime.now());
+ eventHistory.setUpdateUser("TEST");
+ eventHistoryRepository.save(eventHistory);
+ return eventHistory;
+ }
+
  public static AuthorityContact createAuthorityContact() {
  var auth = new AuthorityContact();
  auth.setIndependentAuthorityId(UUID.randomUUID().toString());

tools/config/update-configmap.sh

@@ -53,6 +53,8 @@ echo Creating config map "$APP_NAME"-config-map
 oc create -n "$GRAD_NAMESPACE"-"$envValue" configmap "$APP_NAME"-config-map \
  --from-literal=APP_LOG_LEVEL="$APP_LOG_LEVEL" \
  --from-literal=BASELINE_ON_MIGRATE="true" \
+ --from-literal=CRON_SCHEDULED_PURGE_OLD_RECORDS: "0 0 0 * * *" \
+ --from-literal=RECORDS_STALE_IN_DAYS: 365 \
  --from-literal=CRON_SCHEDULED_GRAD_TO_TRAX_EVENTS="0 0/5 * * * *" \
  --from-literal=CRON_SCHEDULED_GRAD_TO_TRAX_EVENTS_LOCK_AT_LEAST_FOR="PT1M" \
  --from-literal=CRON_SCHEDULED_GRAD_TO_TRAX_EVENTS_LOCK_AT_MOST_FOR="PT5M" \