Merge pull request #134 from pagopa/uat

breaking: promotion to prod - report v2

.github/workflows/release.yml

@@ -17,13 +17,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
         with:
           persist-credentials: false
           fetch-depth: 0
 
       - name: Release
-        uses: cycjimmy/semantic-release-action@8e58d20d0f6c8773181f43eb74d6a05e3099571d # v3.4.2
+        uses: cycjimmy/semantic-release-action@61680d0e9b02ff86f5648ade99e01be17f0260a4 #v4.0.0
         with:
           semantic_version: 18.0.0
           extra_plugins: |

.github/workflows/scan.yml

@@ -33,12 +33,12 @@ jobs:
     environment: dev
     steps:
       - name: Checkout the code
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 #v3.6.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
       - name: Build the Docker image
         run: docker build . --file ${{ env.DOCKERFILE }} --target cve --tag localbuild/testimage:latest
       - name: Run the Trivy scan action itself with GitHub Advanced Security code scanning integration enabled
         id: scan
-        uses: aquasecurity/trivy-action@fbd16365eb88e12433951383f5e99bd901fc618f #v0.12.0
+        uses: aquasecurity/trivy-action@062f2592684a31eb3aa050cc61e7ca1451cecd3d #v0.18.0
         with:
           image-ref: "localbuild/testimage:latest"
           format: 'sarif'

docs/entities.uml

@@ -0,0 +1,40 @@
+@startuml
+class FileReport {
+ -senderCodes: String[]
+ -filesUploaded: FileMetadata[]
+ -ackToDownload: String[]
+ +addFileUploaded(file)
+ +removeFileUploaded(file)
+ +addSenderCode(senderCode)
+ +addAckToDownload(file)
+ +removeAckToDownload(file)
+ +addFileOrUpdateStatus(file)
+ +removeFilesOlderThan(days)
+ +addSquaringDataToFile(file)
+}
+
+class FileMetadata {
+ -name: String
+ -path: String
+ -size: Long
+ -status: Enum
+ -transmissionDate: Date
+ -dataSummary: AggregatesDataSummary
+
+ +enrichWithSquaringData()
+}
+
+class AggregatesDataSummary {
+  -minAccountingDate: Date
+  -maxAccountingDate: Date
+  -numberOfMerchants: Integer
+  -countNegativeTransactions: Long
+  -countPositiveTransactions: Long
+  -sumAmountNegativeTransactions: Long
+  -sumAmountPositiveTransactions: Long
+  -sha256OriginFile: String
+}
+
+FileReport "1" *-- "n" FileMetadata
+FileMetadata *-- AggregatesDataSummary
+@enduml

helm/rtd/values.yaml

@@ -56,6 +56,7 @@ microservice-chart:
     MONGODB_CONNECTION_URI: mongo-db-connection-uri
     KAFKA_RTD_PROJECTOR_SASL_JAAS_CONFIG: evh-rtd-file-register-projector-rtd-file-register-projector-consumer-policy-rtd
     APPLICATIONINSIGHTS_CONNECTION_STRING: appinsights-instrumentation-key
+    INTERNAL_SERVICES_API_KEY: rtd-internal-api-product-subscription-key
 
   envConfigMapExternals:
     rtd-file-register-projector-consumer:
@@ -64,3 +65,4 @@ microservice-chart:
 
     rtd-filereporter:
       OPENTELEMETRY_LOG_LEVEL: APPLICATIONINSIGHTS_INSTRUMENTATION_LOGGING_LEVEL
+      STORAGE_ACCOUNT_HOST: STORAGE_ACCOUNT_HOST

pom.xml

@@ -5,20 +5,20 @@
 	<parent>
 		<groupId>org.springframework.boot</groupId>
 		<artifactId>spring-boot-starter-parent</artifactId>
-		<version>3.2.3</version>
+		<version>3.2.4</version>
 		<relativePath/> <!-- lookup parent from repository -->
 	</parent>
 
 	<groupId>it.gov.pagopa.rtd.ms</groupId>
 	<artifactId>rtd-ms-file-reporter</artifactId>
-	<version>1.4.1</version>
+	<version>2.0.0</version>
 	<name>rtd-ms-file-reporter</name>
 	<description>Micro-service to retrieve and keep updated the file reports for senders</description>
 
 	<properties>
 		<java.version>17</java.version>
 		<spring-cloud.version>2023.0.0</spring-cloud.version>
-		<testcontainers.version>1.19.3</testcontainers.version>
+		<testcontainers.version>1.19.7</testcontainers.version>
 		<snakeyaml.version>2.2</snakeyaml.version>
 		<kafka.version>3.6.1</kafka.version>
 		<sonar.exclusions>**/telemetry/**.java</sonar.exclusions>
@@ -115,6 +115,10 @@
 			<artifactId>mongodb</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.apache.httpcomponents.client5</groupId>
+			<artifactId>httpclient5</artifactId>
+		</dependency>
 
 	</dependencies>
 
@@ -150,12 +154,6 @@
 				<type>pom</type>
 				<scope>import</scope>
 			</dependency>
-<!--			solve CVE-2024-22233 -->
-			<dependency>
-				<groupId>org.springframework</groupId>
-				<artifactId>spring-core</artifactId>
-				<version>6.1.3</version>
-			</dependency>
 		</dependencies>
 	</dependencyManagement>
 

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/RtdMsFileReporterApplication.java

@@ -1,13 +1,16 @@
 package it.gov.pagopa.rtd.ms.rtdmsfilereporter;
 
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.feign.config.StorageProperties;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 
 @SpringBootApplication
+@EnableConfigurationProperties(StorageProperties.class)
 public class RtdMsFileReporterApplication {
 
-	public static void main(String[] args) {
-		SpringApplication.run(RtdMsFileReporterApplication.class, args);
-	}
+  public static void main(String[] args) {
+    SpringApplication.run(RtdMsFileReporterApplication.class, args);
+  }
 
 }

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/configuration/AppConfiguration.java

@@ -2,6 +2,7 @@
 
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.FileReportCommandFactory;
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.repository.FileReportRepository;
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.service.DecryptedEventCommand;
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.service.FileReportService;
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.service.FileReportServiceImpl;
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.persistance.FileReportDao;
@@ -23,12 +24,13 @@ public FileReportService getFileReportService(FileReportRepository repository) {
   }
 
   @Bean
-  public FileReportRepository getFileReportRepository(FileReportDao dao, FileReportEntityMapper mapper) {
+  public FileReportRepository getFileReportRepository(FileReportDao dao,
+      FileReportEntityMapper mapper) {
     return new FileReportRepositoryImpl(dao, mapper);
   }
 
   @Bean
-  public FileReportCommandFactory getFileReportCommandFactory() {
-    return new FileReportCommandFactory();
+  public FileReportCommandFactory getFileReportCommandFactory(DecryptedEventCommand command) {
+    return new FileReportCommandFactory(command);
   }
 }

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/controller/ConsumeEventControllerImpl.java

@@ -3,8 +3,8 @@
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.event.FileReportEventAdapter;
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.event.model.ProjectorEventDto;
 import jakarta.validation.Valid;
+import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.integration.support.MessageBuilder;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.ResponseBody;
@@ -13,14 +13,14 @@
 @RestController
 @ResponseBody
 @Slf4j
+@RequiredArgsConstructor
 public class ConsumeEventControllerImpl implements ConsumeEventController {
 
-  @Autowired
-  private FileReportEventAdapter adapter;
+  private final FileReportEventAdapter adapter;
 
   @Override
   public void consumeEvent(@RequestBody @Valid ProjectorEventDto eventData) {
-    log.info("Received event [{}]", eventData.getFileName());
+    log.info("Received event from controller");
 
     adapter.consumeEvent(MessageBuilder.withPayload(eventData).build().getPayload());
   }

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/controller/FileReportController.java

@@ -1,7 +1,8 @@
 package it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller;
 
-import it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.FileReportDto;
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.SenderAdeAckListDto;
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v1.FileReportDto;
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v2.FileReportV2Dto;
 import jakarta.validation.constraints.NotNull;
 import java.util.Collection;
 import org.springframework.http.MediaType;
@@ -13,6 +14,10 @@ public interface FileReportController {
   @GetMapping(value = "file-report", produces = MediaType.APPLICATION_JSON_VALUE)
   FileReportDto getFileReport(@RequestParam(name = "senderCodes") Collection<String> senderCodes);
 
+  @GetMapping(value = "v2/file-report", produces = MediaType.APPLICATION_JSON_VALUE)
+  FileReportV2Dto getFileReportV2(
+      @RequestParam(name = "senderCodes") Collection<String> senderCodes);
+
   @GetMapping(value = "/sender-ade-ack", produces = MediaType.APPLICATION_JSON_VALUE)
   SenderAdeAckListDto getSenderAdeAckList(
       @NotNull @RequestParam(name = "senderCodes") Collection<String> senderCodes);

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/controller/FileReportControllerImpl.java

@@ -1,8 +1,10 @@
 package it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller;
 
-import it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.FileReportDto;
-import it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.FileReportDtoMapper;
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.SenderAdeAckListDto;
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v1.FileReportDto;
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v1.FileReportDtoMapper;
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v2.FileReportV2Dto;
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v2.FileReportV2DtoMapper;
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.service.FileReportService;
 import jakarta.validation.constraints.NotNull;
 import java.util.Collection;
@@ -17,13 +19,20 @@ public class FileReportControllerImpl implements FileReportController {
 
   private final FileReportService fileReportService;
   private final FileReportDtoMapper mapper;
+  private final FileReportV2DtoMapper mapperV2;
 
   @Override
   public FileReportDto getFileReport(Collection<String> senderCodes) {
     log.info("GET file report for sender codes: {}", senderCodes.toString());
     return mapper.fileReportToDto(fileReportService.getAggregateFileReport(senderCodes));
   }
 
+  @Override
+  public FileReportV2Dto getFileReportV2(Collection<String> senderCodes) {
+    log.info("GET file report v2 for sender codes: {}", senderCodes.toString());
+    return mapperV2.fileReportToDto(fileReportService.getAggregateFileReport(senderCodes));
+  }
+
   @Override
   public SenderAdeAckListDto getSenderAdeAckList(@NotNull Collection<String> senderCodes) {
     log.info("GET ack to download list for sender codes: {}", senderCodes.toString());

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/controller/model/FileMetadataDto.java → src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/controller/model/v1/FileMetadataDto.java

@@ -1,4 +1,4 @@
-package it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model;
+package it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v1;
 
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/controller/model/FileReportDto.java → src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/controller/model/v1/FileReportDto.java

@@ -1,4 +1,4 @@
-package it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model;
+package it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v1;
 
 import java.util.Collection;
 import lombok.Data;

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/controller/model/FileReportDtoMapper.java → src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/controller/model/v1/FileReportDtoMapper.java

@@ -1,4 +1,4 @@
-package it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model;
+package it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v1;
 
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.model.FileReport;
 import org.mapstruct.Mapper;

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/controller/model/v2/FileMetadataV2Dto.java

@@ -0,0 +1,15 @@
+package it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v2;
+
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v1.FileMetadataDto;
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.model.AggregatesDataSummary;
+import jakarta.validation.constraints.NotNull;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@EqualsAndHashCode(callSuper = true)
+@Data
+public class FileMetadataV2Dto extends FileMetadataDto {
+
+  @NotNull
+  private AggregatesDataSummary dataSummary;
+}

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/controller/model/v2/FileReportV2Dto.java

@@ -0,0 +1,10 @@
+package it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v2;
+
+import java.util.Collection;
+import lombok.Data;
+
+@Data
+public class FileReportV2Dto {
+
+  Collection<FileMetadataV2Dto> filesRecentlyUploaded;
+}

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/controller/model/v2/FileReportV2DtoMapper.java

@@ -0,0 +1,16 @@
+package it.gov.pagopa.rtd.ms.rtdmsfilereporter.controller.model.v2;
+
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.model.FileMetadata;
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.model.FileReport;
+import org.mapstruct.Mapper;
+import org.mapstruct.Mapping;
+
+@Mapper(componentModel = "spring")
+public interface FileReportV2DtoMapper {
+
+  @Mapping(source = "filesUploaded", target = "filesRecentlyUploaded")
+  FileReportV2Dto fileReportToDto(FileReport fileReport);
+
+  @Mapping(source = "aggregatesDataSummary", target = "dataSummary")
+  FileMetadataV2Dto fileMetadataToDto(FileMetadata fileMetadata);
+}

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/domain/FileReportCommandFactory.java

@@ -2,18 +2,24 @@
 
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.model.FileMetadata;
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.model.FileReport;
+import it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.service.DecryptedEventCommand;
 import it.gov.pagopa.rtd.ms.rtdmsfilereporter.event.model.EventStatusEnum;
 import java.util.function.BiConsumer;
+import lombok.RequiredArgsConstructor;
 
+@RequiredArgsConstructor
 public class FileReportCommandFactory {
 
+  private final DecryptedEventCommand decryptCommand;
+
   public BiConsumer<FileReport, FileMetadata> getCommandByStatus(String status) {
     return switch (EventStatusEnum.valueOf(status)) {
       case ACK_TO_DOWNLOAD ->
           (fileReport, fileMetadata) -> fileReport.addAckToDownload(fileMetadata.getName());
       case ACK_DOWNLOADED ->
           (fileReport, fileMetadata) -> fileReport.removeAckToDownload(fileMetadata.getName());
-      default -> FileReport::addFileOrUpdateStatusIfPresent;
+      case DECRYPTED -> decryptCommand;
+      case RECEIVED, SENT_TO_ADE -> FileReport::addFileOrUpdateStatusIfPresent;
     };
   }
 }

src/main/java/it/gov/pagopa/rtd/ms/rtdmsfilereporter/domain/model/AggregatesDataSummary.java

@@ -0,0 +1,36 @@
+package it.gov.pagopa.rtd.ms.rtdmsfilereporter.domain.model;
+
+import java.time.LocalDate;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class AggregatesDataSummary {
+
+  private LocalDate minAccountingDate;
+  private LocalDate maxAccountingDate;
+  private int numberOfMerchants;
+  private long countNegativeTransactions;
+  private long countPositiveTransactions;
+  private long sumAmountNegativeTransactions;
+  private long sumAmountPositiveTransactions;
+  // sha256 of the initial input file containing the transactions
+  private String sha256OriginFile;
+
+  public static AggregatesDataSummary createInvalidDataSummary() {
+    return AggregatesDataSummary.builder()
+        .sumAmountPositiveTransactions(-1)
+        .sumAmountNegativeTransactions(-1)
+        .countPositiveTransactions(-1)
+        .countNegativeTransactions(-1)
+        .numberOfMerchants(-1)
+        .minAccountingDate(null)
+        .maxAccountingDate(null)
+        .build();
+  }
+}