Skip to content

Commit

Permalink
#1692 GET $apiUrl/{datasetName} and s"GET $apiUrl/{datasetName}/{da…
Browse files Browse the repository at this point in the history 
…tasetVersion} are checked for ds existence => 404
  • Loading branch information
@dk1844
dk1844 committed Jun 20, 2022
1 parent b252ecf commit 3db75dd
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 7 deletions.
4 changes: 2 additions & 2 deletions rest-api/src/main/scala/za/co/absa/enceladus/rest_api/controllers/v3/RunControllerV3.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,8 @@ class RunControllerV3 @Autowired()(runService: RunServiceV3) extends BaseControl
@PathVariable datasetVersion: Int,
@RequestParam startDate: Optional[String]): CompletableFuture[Seq[RunSummary]] = {
runService.getRunSummaries(
datasetName = Some(datasetName),
datasetVersion = Some(datasetVersion),
datasetName = datasetName,
datasetVersion = datasetVersion,
startDate = parseYmdDate(startDate)
)
}
Expand Down
19 changes: 14 additions & 5 deletions rest-api/src/main/scala/za/co/absa/enceladus/rest_api/services/v3/RunServiceV3.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,18 +57,27 @@ class RunServiceV3 @Autowired()(runMongoRepository: RunMongoRepositoryV3, datase
* @return
*/
def getLatestOfEachRunSummary(datasetName: Option[String] = None,
datasetVersion: Option[Int] = None,
startDate: Option[LocalDate] = None,
sparkAppId: Option[String] = None,
uniqueId: Option[String] = None
): Future[Seq[RunSummary]] = {
runMongoRepository.getRunSummariesLatestOfEach(datasetName, datasetVersion, startDate, sparkAppId, uniqueId)
datasetName match {
case None => runMongoRepository.getRunSummariesLatestOfEach(None, None, startDate, sparkAppId, uniqueId)
case definedDsName @ Some(dsName) => datasetServiceV3.getLatestVersion(dsName).flatMap {
case None => Future.failed(NotFoundException(s"Dataset $datasetName at all."))
case Some(_) => runMongoRepository.getRunSummariesLatestOfEach(definedDsName, None, startDate, sparkAppId, uniqueId)

}
}
}

def getRunSummaries(datasetName: Option[String] = None,
datasetVersion: Option[Int] = None,
def getRunSummaries(datasetName: String,
datasetVersion: Int,
startDate: Option[LocalDate] = None): Future[Seq[RunSummary]] = {
runMongoRepository.getRunSummaries(datasetName, datasetVersion, startDate)
datasetServiceV3.getVersion(datasetName, datasetVersion).flatMap {
case Some(_) => runMongoRepository.getRunSummaries(Some(datasetName), Some(datasetVersion), startDate)
case _ => Future.failed(NotFoundException(s"Dataset $datasetName v$datasetVersion does not exist."))
}
}

override def addCheckpoint(datasetName: String, datasetVersion: Int, runId: Int, newCheckpoint: Checkpoint): Future[Run] = {
Expand Down
71 changes: 71 additions & 0 deletions .../absa/enceladus/rest_api/integration/controllers/v3/RunControllerV3IntegrationSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,11 @@ class RunControllerV3IntegrationSuite extends BaseRestApiTestV3 with Matchers {
s"GET $apiUrl/{datasetName}" can {
"return 200" when {
"latest RunSummaries are queried" in {
datasetFixture.add(
DatasetFactory.getDummyDataset("dataset1", version = 1),
DatasetFactory.getDummyDataset("dataset1", version = 2),
DatasetFactory.getDummyDataset("dataset2", version = 1)
)
val dataset1ver1run1 = RunFactory.getDummyRun(dataset = "dataset1", datasetVersion = 1, runId = 1)
val dataset1ver1run2 = RunFactory.getDummyRun(dataset = "dataset1", datasetVersion = 1, runId = 2)
runFixture.add(dataset1ver1run1, dataset1ver1run2)
Expand All @@ -171,6 +176,13 @@ class RunControllerV3IntegrationSuite extends BaseRestApiTestV3 with Matchers {
}

"latest RunSummaries are queried on startDate" in {
datasetFixture.add(
DatasetFactory.getDummyDataset("dataset1", version = 1),
DatasetFactory.getDummyDataset("dataset1", version = 2),
DatasetFactory.getDummyDataset("dataset2", version = 1),
DatasetFactory.getDummyDataset("dataset3", version = 1)
)

val dataset1ver1run1 = RunFactory.getDummyRun(dataset = "dataset1", datasetVersion = 1, runId = 1, startDateTime = "30-01-2022 13:01:12 +0200")
val dataset1ver1run2 = RunFactory.getDummyRun(dataset = "dataset1", datasetVersion = 1, runId = 2, startDateTime = "22-05-2022 14:01:12 +0200")

Expand Down Expand Up @@ -207,6 +219,10 @@ class RunControllerV3IntegrationSuite extends BaseRestApiTestV3 with Matchers {
}

"latest RunSummaries are queried, but nothing is found" in {
datasetFixture.add(
DatasetFactory.getDummyDataset("dataset1", version = 1),
DatasetFactory.getDummyDataset("dataset3", version = 1)
)
val run1 = RunFactory.getDummyRun(dataset = "dataset1", startDateTime = "22-05-2022 14:01:12 +0200")
val run2 = RunFactory.getDummyRun(dataset = "dataset3", uniqueId = None) // unrelated to dataset1
runFixture.add(run1, run2)
Expand All @@ -215,13 +231,40 @@ class RunControllerV3IntegrationSuite extends BaseRestApiTestV3 with Matchers {
response.getStatusCode shouldBe HttpStatus.OK
response.getBody shouldBe "[]" // empty array
}
"return RunSummaries by dataset name - ok even for no runs for known dataset" in {
// datasets referenced by runs must exist, too
datasetFixture.add(
DatasetFactory.getDummyDataset("dataset1", version = 1)
)

val response = sendGet[Array[RunSummary]](s"$apiUrl/dataset1")
response.getStatusCode shouldBe HttpStatus.OK

val expected = List.empty[RunSummary]
response.getBody shouldBe expected
}
}

"return 404" when {
"RunSummaries for non-existent dataset name is queried" in {
// datasets referenced by runs must exist
val response = sendGet[Array[RunSummary]](s"$apiUrl/dataset1")
response.getStatusCode shouldBe HttpStatus.NOT_FOUND
}
}

}

s"GET $apiUrl/{datasetName}/{datasetVersion}" can {
"return 200" when {
"return RunSummaries by dataset name and version" in {
// datasets referenced by runs must exist, too
datasetFixture.add(
DatasetFactory.getDummyDataset("dataset1", version = 1),
DatasetFactory.getDummyDataset("dataset1", version = 2),
DatasetFactory.getDummyDataset("dataset2", version = 1)
)

val dataset1ver1run1 = RunFactory.getDummyRun(dataset = "dataset1", datasetVersion = 1, runId = 1)
val dataset1ver1run2 = RunFactory.getDummyRun(dataset = "dataset1", datasetVersion = 1, runId = 2)
runFixture.add(dataset1ver1run1, dataset1ver1run2)
Expand All @@ -237,6 +280,13 @@ class RunControllerV3IntegrationSuite extends BaseRestApiTestV3 with Matchers {
}

"return RunSummaries on combination of (startDate, dsName, and dsVersion)" in {
// datasets referenced by runs must exist, too
datasetFixture.add(
DatasetFactory.getDummyDataset("dataset1", version = 1),
DatasetFactory.getDummyDataset("dataset1", version = 2),
DatasetFactory.getDummyDataset("dataset3", version = 1)
)

val dataset1ver1run2 = RunFactory.getDummyRun(dataset = "dataset1", datasetVersion = 1, runId = 2, startDateTime = "22-05-2022 14:01:12 +0200")

val dataset1ver2run1 = RunFactory.getDummyRun(dataset = "dataset1", datasetVersion = 2, runId = 1, startDateTime = "30-01-2022 15:01:12 +0200")
Expand All @@ -258,6 +308,27 @@ class RunControllerV3IntegrationSuite extends BaseRestApiTestV3 with Matchers {
response.getBody shouldBe expected

}

"return RunSummaries by dataset name and version - ok even for no runs for known dataset" in {
// datasets referenced by runs must exist, too
datasetFixture.add(
DatasetFactory.getDummyDataset("dataset1", version = 1)
)

val response = sendGet[Array[RunSummary]](s"$apiUrl/dataset1/1")
response.getStatusCode shouldBe HttpStatus.OK

val expected = List.empty[RunSummary]
response.getBody shouldBe expected
}
}

"return 404" when {
"RunSummaries for non-existent dataset name and version are queried" in {
// datasets referenced by runs must exist
val response = sendGet[Array[RunSummary]](s"$apiUrl/dataset1/1")
response.getStatusCode shouldBe HttpStatus.NOT_FOUND
}
}
}

Expand Down

0 comments on commit 3db75dd

Please sign in to comment.