Add Check that Exported CSV has correct amount of entities (#278)

src/main/java/app/coronawarn/quicktest/archive/repository/ArchiveRepository.java

@@ -138,4 +138,23 @@ public List<String> findAllHashedGuids(final List<String> search) {
  this.em.getTransaction().commit();
  return result;
  }
+
+ /**
+ * Count entities by tenantId.
+ *
+ * @param tenantId SHA256 Hash of TenantId to search for
+ * @return amount of found entities.
+ */
+ public Integer countAllByTenantId(final String tenantId) {
+ em.getTransaction().begin();
+
+ final Integer result = em
+ .createQuery("SELECT COUNT(*) FROM Archive a WHERE a.tenantId = ?1", Long.class)
+ .setParameter(1, tenantId)
+ .getSingleResult()
+ .intValue();
+
+ em.getTransaction().commit();
+ return result;
+ }
 }

src/main/java/app/coronawarn/quicktest/domain/Cancellation.java

@@ -80,6 +80,10 @@ public class Cancellation {
  @JsonIgnore
  private Integer csvEntityCount;
 
+ @Column(name = "db_entity_count")
+ @JsonIgnore
+ private Integer dbEntityCount;
+
  @Column(name = "csv_hash")
  @JsonIgnore
  private String csvHash;

src/main/java/app/coronawarn/quicktest/service/ArchiveService.java

@@ -233,6 +233,16 @@ public void deleteByTenantId(String partnerId) {
  longTermArchiveRepository.deleteAllByTenantId(createHash(partnerId));
  }
 
+ /**
+ * Counts the existing entities in Long Term Archive by given TenantId.
+ *
+ * @param tenantId Tenant ID to search for.
+ * @return Amount of found entities
+ */
+ public Integer countByTenantId(String tenantId) {
+ return longTermArchiveRepository.countAllByTenantId(createHash(tenantId));
+ }
+
  private ArchiveCipherDtoV1 convertQuickTest(final QuickTestArchiveDataView quickTestArchive) {
  final ArchiveCipherDtoV1 archive = new ArchiveCipherDtoV1();
 

src/main/java/app/coronawarn/quicktest/service/CancellationSchedulingService.java

@@ -75,7 +75,8 @@ private void processCancellationArchiveBatchRecursion(List<Cancellation> cancell
  for (Cancellation cancellation : cancellations) {
  String partnerId = cancellation.getPartnerId();
  archiveService.moveToArchiveByTenantId(partnerId);
- cancellationService.updateMovedToLongterm(cancellation, ZonedDateTime.now());
+ Integer entityCount = archiveService.countByTenantId(partnerId);
+ cancellationService.updateMovedToLongterm(cancellation, ZonedDateTime.now(), entityCount);
  }
 
  List<Cancellation> nextBatch = cancellationService.getReadyToArchiveBatch();
@@ -148,8 +149,17 @@ private void processCsvUploadBatchRecursion(List<Cancellation> cancellations) {
  log.info("File stored to S3 with id: {}, size: {}, hash: {}",
  objectId, csvBytes.length, getHash(csvBytes));
 
- cancellationService.updateCsvCreated(cancellation, ZonedDateTime.now(), objectId,
- getHash(csvBytes), totalEntityCount, csvBytes.length);
+ if (cancellation.getDbEntityCount() == totalEntityCount) {
+ cancellationService.updateCsvCreated(cancellation, ZonedDateTime.now(), objectId,
+ getHash(csvBytes), totalEntityCount, csvBytes.length);
+ } else {
+ log.error("Difference between actual and expected EntityCount in CSV File for partner {}. "
+ + "Expected: {}, Acutal: {}, CSV Export will not be marked as finished.",
+ cancellation.getPartnerId(), cancellation.getDbEntityCount(), totalEntityCount);
+
+ cancellationService.updateDataExportError(cancellation, "CSV Export Delta detected. "
+ + "Expected: " + cancellation.getDbEntityCount() + " Actual: " + totalEntityCount);
+ }
  } catch (Exception e) {
  String errorMessage = e.getClass().getName() + ": " + e.getMessage();
 

src/main/java/app/coronawarn/quicktest/service/CancellationService.java

@@ -109,8 +109,10 @@ public Optional<Cancellation> getByPartnerId(String partnerId) {
  * @param cancellation Cancellation Entity
  * @param movedToLongtermArchive timestamp of job completion
  */
- public void updateMovedToLongterm(Cancellation cancellation, ZonedDateTime movedToLongtermArchive) {
+ public void updateMovedToLongterm(Cancellation cancellation, ZonedDateTime movedToLongtermArchive,
+ Integer entityCount) {
  cancellation.setMovedToLongtermArchive(movedToLongtermArchive);
+ cancellation.setDbEntityCount(entityCount);
  cancellationRepository.save(cancellation);
  }
 

src/main/resources/db/changelog.yml

@@ -71,3 +71,6 @@ databaseChangeLog:
  - include:
  file: changelog/V024_update_cancellationTable_5.yml
  relativeToChangelogFile: true
+ - include:
+ file: changelog/V025_update_cancellationTable_6.yml
+ relativeToChangelogFile: true

src/main/resources/db/changelog/V025_update_cancellationTable_6.yml

@@ -0,0 +1,12 @@
+databaseChangeLog:
+ - changeSet:
+ id: update-cancellation-table-6
+ author: f11h
+ changes:
+ - addColumn:
+ tableName: cancellation
+ column:
+ name: db_entity_count
+ type: int
+ constraints:
+ nullable: true

src/test/java/app/coronawarn/quicktest/service/CancellationCsvTest.java

@@ -49,18 +49,21 @@
 import java.time.LocalDateTime;
 import java.time.ZonedDateTime;
 import java.time.temporal.ChronoUnit;
+import java.util.Collections;
 import java.util.List;
 import org.apache.commons.lang3.RandomUtils;
 import org.apache.tomcat.util.buf.HexUtils;
 import org.bouncycastle.util.encoders.Hex;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.mockito.ArgumentCaptor;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.boot.test.mock.mockito.SpyBean;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
 import org.springframework.transaction.annotation.Transactional;
@@ -83,6 +86,9 @@ class CancellationCsvTest {
  @MockBean
  private AmazonS3 s3Client;
 
+ @SpyBean
+ private CancellationService cancellationServiceSpy;
+
  @BeforeEach
  void setUp() {
  shortTermArchiveRepository.deleteAll();
@@ -114,6 +120,7 @@ void testCsvExport(int n) throws IOException, NoSuchAlgorithmException, CsvExcep
 
  Assertions.assertEquals(0, shortTermArchiveRepository.findAllByTenantId(PARTNER_ID, Pageable.unpaged()).count());
  Assertions.assertEquals(n, longTermArchiveRepository.findAllByTenantId(PARTNER_ID_HASH, PageRequest.of(0, Integer.MAX_VALUE)).size());
+ Assertions.assertEquals(n, cancellationRepository.findById(PARTNER_ID).orElseThrow().getDbEntityCount());
 
  ArgumentCaptor<InputStream> inputStreamArgumentCaptor = ArgumentCaptor.forClass(InputStream.class);
  String expectedFileName = PARTNER_ID + ".csv";
@@ -151,6 +158,42 @@ void testCsvExport(int n) throws IOException, NoSuchAlgorithmException, CsvExcep
  longTermArchiveRepository.deleteAllByTenantId(PARTNER_ID_HASH);
  }
 
+ @Transactional
+ @Test
+ void testCsvExportEntityCountDelta() {
+ Cancellation cancellation = new Cancellation();
+ cancellation.setPartnerId(PARTNER_ID);
+ cancellation.setCancellationDate(CANCELLATION_DATE);
+ cancellationRepository.save(cancellation);
+
+ // Import 10 Test Entities
+ for (int i = 0; i < 10; i++) {
+ shortTermArchiveRepository.save(buildQuickTestArchive(PARTNER_ID));
+ }
+
+ // Execute Archiving
+ cancellationSchedulingService.cancellationArchiveJob();
+
+ // Manipulate Entity Count by +1
+ cancellation = cancellationRepository.findById(PARTNER_ID).orElseThrow();
+ cancellation.setDbEntityCount(cancellation.getDbEntityCount() + 1);
+
+ when(cancellationServiceSpy.getReadyToUploadBatch())
+ .thenCallRealMethod()
+ .thenReturn(Collections.emptyList());
+ when(s3Client.putObject(anyString(), anyString(), any(), any())).thenReturn(new PutObjectResult());
+
+ cancellationSchedulingService.csvUploadJob();
+
+ // Check that CSV Created Timestamp was not set
+ cancellation = cancellationRepository.findById(PARTNER_ID).orElseThrow();
+ Assertions.assertNull(cancellation.getCsvCreated());
+ Assertions.assertEquals("CSV Export Delta detected. Expected: 11 Actual: 10", cancellation.getDataExportError());
+
+ // Cleanup
+ longTermArchiveRepository.deleteAllByTenantId(PARTNER_ID_HASH);
+ }
+
  private String getHash(byte[] bytes) throws NoSuchAlgorithmException {
  return Hex.toHexString(MessageDigest.getInstance("SHA-256").digest(bytes));
  }

src/test/java/app/coronawarn/quicktest/service/CancellationServiceTest.java

@@ -148,7 +148,7 @@ void testUpdateMovedToLongtermArchive() {
  Cancellation cancellation = cancellationService.createCancellation(PARTNER_ID, CANCELLATION_DATE);
  Assertions.assertEquals(FINAL_DELETION, cancellation.getFinalDeletion());
 
- cancellationService.updateMovedToLongterm(cancellation, NEW_STATE_DATE);
+ cancellationService.updateMovedToLongterm(cancellation, NEW_STATE_DATE, 42);
 
  Optional<Cancellation> updatedCancellation = cancellationRepository.findById(PARTNER_ID);
  Assertions.assertTrue(updatedCancellation.isPresent());
@@ -160,6 +160,7 @@ void testUpdateMovedToLongtermArchive() {
  assertNull(updatedCancellation.get().getBucketObjectId());
  assertNull(updatedCancellation.get().getDownloadLinkRequested());
  Assertions.assertEquals(NEW_STATE_DATE, updatedCancellation.get().getMovedToLongtermArchive());
+ Assertions.assertEquals(42, updatedCancellation.get().getDbEntityCount());
  }
 
  @Test
@@ -292,7 +293,7 @@ void testGetReadyToArchiveYoungerThen48h() {
  void testGetReadyToArchiveMovedNotNull() {
  Cancellation cancellation = cancellationService.createCancellation(PARTNER_ID, CANCELLATION_DATE);
  Assertions.assertEquals(FINAL_DELETION, cancellation.getFinalDeletion());
- cancellationService.updateMovedToLongterm(cancellation, ZonedDateTime.now());
+ cancellationService.updateMovedToLongterm(cancellation, ZonedDateTime.now(), 0);
  List<Cancellation> results = cancellationService.getReadyToArchiveBatch();
  assertTrue(results.isEmpty());
  }