Feature/1692 api v3 runs (#2073)

* #1692 - runs serialization based on Atum 3.8.2 - impl: API: RunControllerV3, persistence: RunMongoRepositoryV3.{getLatestOfEachRunSummary, ...} + IT - jackson annotations exclusions to use 2.40.4 - filtering runs from `startDate` uses LocalDate/ISODate instead of lexicographical comparison as v2 (V2 uses DD-MM-YYYY). Input changed for controllers - accepts **YYYY-MM-DD**. - `GET $apiUrl/{datasetName}` and s"GET $apiUrl/{datasetName}/{datasetVersion} are checked for ds existence => 404 Co-authored-by: David Benedeki <[email protected]> Co-authored-by: Adrian Olosutean <[email protected]> Co-authored-by: Saša Zejnilović <[email protected]>
AbsaOSS · Jun 23, 2022 · c398997 · c398997
1 parent efb2990
commit c398997
Show file tree

Hide file tree

Showing 25 changed files with 1,787 additions and 103 deletions.
diff --git a/dao/src/main/scala/za/co/absa/enceladus/dao/MenasDAO.scala b/dao/src/main/scala/za/co/absa/enceladus/dao/MenasDAO.scala
@@ -16,8 +16,8 @@
 package za.co.absa.enceladus.dao
 
 import org.apache.spark.sql.types.StructType
+import za.co.absa.atum.model.{Checkpoint, ControlMeasure, RunStatus}
 import za.co.absa.enceladus.model._
-import za.co.absa.atum.model._
 import za.co.absa.enceladus.utils.validation.ValidationLevel.Constants.DefaultValidationLevel
 import za.co.absa.enceladus.utils.validation.ValidationLevel.ValidationLevel
 

diff --git a/dao/src/test/scala/za/co/absa/enceladus/dao/rest/JsonSerializerSuite.scala b/dao/src/test/scala/za/co/absa/enceladus/dao/rest/JsonSerializerSuite.scala
@@ -575,10 +575,7 @@ class JsonSerializerSuite extends BaseTestSuite with VersionedModelMatchers {
            |  },
            |  "startDateTime": "04-12-2017 16:19:17 +0200",
            |  "runStatus": {
-           |    "status": {
-           |      "enumClass": "za.co.absa.atum.model.RunState",
-           |      "value": "allSucceeded"
-           |    },
+           |    "status": "allSucceeded",
            |    "error": null
            |  },
            |  "controlMeasure": {

diff --git a/data-model/src/main/scala/za/co/absa/enceladus/model/Run.scala b/data-model/src/main/scala/za/co/absa/enceladus/model/Run.scala
@@ -16,7 +16,6 @@
 package za.co.absa.enceladus.model
 
 import za.co.absa.atum.model.{ControlMeasure, RunStatus}
-import com.typesafe.config.{Config, ConfigFactory}
 
 case class Run
 (

diff --git a/migrations/src/main/scala/za/co/absa/enceladus/migrations/migrations/MigrationToV1.scala b/migrations/src/main/scala/za/co/absa/enceladus/migrations/migrations/MigrationToV1.scala
@@ -40,6 +40,7 @@ object MigrationToV1 extends MigrationBase with CollectionMigration with JsonMig
   createIndex("schema", Seq(IndexField("name", ASC), IndexField("version", ASC)), unique = true)
   createIndex("mapping_table", Seq(IndexField("name", ASC), IndexField("version", ASC)), unique = true)
   createIndex("run", Seq(IndexField("dataset", ASC), IndexField("datasetVersion", ASC), IndexField("runId", ASC)), unique = true)
+  createIndex("run", Seq(IndexField("runId", ASC)))
   createIndex("run", Seq(IndexField("uniqueId", ASC)), unique = true, sparse = true)
   createIndex("attachment", Seq(IndexField("refName", ASC), IndexField("refVersion", ASC)))
 

diff --git a/pom.xml b/pom.xml
@@ -142,7 +142,7 @@
         <abris.version>3.1.1</abris.version>
         <absa.commons.version>1.0.0</absa.commons.version>
         <absa.spark.commons.version>0.2.0</absa.spark.commons.version>
-        <atum.version>3.7.0</atum.version>
+        <atum.version>3.8.2</atum.version>
         <bower.chart.js.version>2.7.3</bower.chart.js.version>
         <bson.codec.jsr310.version>3.5.4</bson.codec.jsr310.version>
         <cobrix.version>2.4.2</cobrix.version>
@@ -154,7 +154,7 @@
         <httpclient.version>4.4.1</httpclient.version>
         <jackson.spark.datatype.version>2.10.4</jackson.spark.datatype.version>
         <jackson.spark.version>2.10.4</jackson.spark.version>
-        <jackson.version>2.9.8</jackson.version>
+        <jackson.version>2.10.4</jackson.version>
         <jjwt.version>0.10.7</jjwt.version>
         <json4s.version>3.5.3</json4s.version>
         <junit.version>4.11</junit.version>
@@ -269,6 +269,15 @@
                     <groupId>org.apache.httpcomponents</groupId>
                     <artifactId>httpclient</artifactId>
                 </exclusion>
+                <!-- in order to use higher version instead of builtin 2.6.7 -->
+                <exclusion>
+                    <artifactId>jackson-databind</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>jackson-annotations</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -285,6 +294,10 @@
                     <groupId>org.apache.httpcomponents</groupId>
                     <artifactId>httpclient</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

diff --git a/rest-api/pom.xml b/rest-api/pom.xml
@@ -63,6 +63,15 @@
                     <groupId>javax.validation</groupId>
                     <artifactId>validation-api</artifactId>
                 </exclusion>
+                <!-- in order to use higher version instead of builtin 2.6.7 -->
+                <exclusion>
+                    <artifactId>jackson-databind</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                </exclusion>
+                <exclusion>
+                    <artifactId>jackson-annotations</artifactId>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
         <!-- Spark-avro for AvroSchema conversions -->

diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/Application.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/Application.scala
@@ -15,6 +15,7 @@
 
 package za.co.absa.enceladus.rest_api
 
+import com.fasterxml.jackson.annotation.JsonInclude.Include
 import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper, SerializationFeature}
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
@@ -52,6 +53,11 @@ class Application() {
       .registerModule(new JavaTimeModule())
       .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false)
       .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+
+      // todo consider including the following to unify serialization results - #2074
+      // .setSerializationInclusion(Include.NON_ABSENT)
+      // ^ fields of `Option[T]` are not included if None (ame behavior as Atum's SerializationUtils.asJson)
+      // explanation: https://github.com/FasterXML/jackson-module-scala/issues/46#issuecomment-128770969
   }
 }
 

diff --git a/...-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/LandingPageController.scala b/...-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/LandingPageController.scala
@@ -16,7 +16,6 @@
 package za.co.absa.enceladus.rest_api.controllers
 
 import java.util.concurrent.CompletableFuture
-
 import scala.concurrent.Future
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.scheduling.annotation.Async
@@ -36,11 +35,11 @@ import za.co.absa.enceladus.rest_api.services.RunService
 @RestController
 @RequestMapping(Array("/api/landing"))
 class LandingPageController @Autowired() (datasetRepository: DatasetMongoRepository,
-    mappingTableRepository: MappingTableMongoRepository,
-    schemaRepository: SchemaMongoRepository,
-    runsService: RunService,
-    landingPageRepository: LandingPageStatisticsMongoRepository,
-    statisticsService: StatisticsService) extends BaseController {
+                                          mappingTableRepository: MappingTableMongoRepository,
+                                          schemaRepository: SchemaMongoRepository,
+                                          runService: RunService,
+                                          landingPageRepository: LandingPageStatisticsMongoRepository,
+                                          statisticsService: StatisticsService) extends BaseController {
 
   import scala.concurrent.ExecutionContext.Implicits.global
   import za.co.absa.enceladus.rest_api.utils.implicits._
@@ -54,7 +53,7 @@ class LandingPageController @Autowired() (datasetRepository: DatasetMongoReposit
     val dsCountFuture = datasetRepository.distinctCount()
     val mappingTableFuture = mappingTableRepository.distinctCount()
     val schemaFuture = schemaRepository.distinctCount()
-    val runFuture = runsService.getCount()
+    val runFuture = runService.getCount()
     val propertiesWithMissingCountsFuture = statisticsService.getPropertiesWithMissingCount()
     val propertiesTotalsFuture: Future[(Int, Int, Int)] = propertiesWithMissingCountsFuture.map(props => {
       props.foldLeft(0, 0, 0) { (acum, item) =>
@@ -66,7 +65,7 @@ class LandingPageController @Autowired() (datasetRepository: DatasetMongoReposit
         }
       }
     })
-    val todaysStatsfuture = runsService.getTodaysRunsStatistics()
+    val todaysStatsfuture = runService.getTodaysRunsStatistics()
     for {
       dsCount <- dsCountFuture
       mtCount <- mappingTableFuture

diff --git a/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RunController.scala b/rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/RunController.scala
@@ -16,7 +16,6 @@
 package za.co.absa.enceladus.rest_api.controllers
 
 import java.util.concurrent.CompletableFuture
-
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.http.HttpStatus
 import org.springframework.security.core.annotation.AuthenticationPrincipal
@@ -63,13 +62,13 @@ class RunController @Autowired()(runService: RunService) extends BaseController
   @GetMapping(Array("/grouped"))
   @ResponseStatus(HttpStatus.OK)
   def getRunSummariesPerDatasetName(): CompletableFuture[Seq[RunDatasetNameGroupedSummary]] = {
-    runService.getRunSummariesPerDatasetName()
+    runService.getGroupedRunSummariesPerDatasetName()
   }
 
   @GetMapping(Array("/grouped/{datasetName}"))
   @ResponseStatus(HttpStatus.OK)
   def getRunSummariesPerDatasetVersion(@PathVariable datasetName: String): CompletableFuture[Seq[RunDatasetVersionGroupedSummary]] = {
-    runService.getRunSummariesPerDatasetVersion(datasetName)
+    runService.getGroupedRunSummariesPerDatasetVersion(datasetName)
   }
 
   @GetMapping(Array("/bySparkAppId/{appId}"))