From ed08f8234c33386c47265dce69606e660a7026cf Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Mon, 21 Aug 2023 10:00:41 +0200 Subject: [PATCH] Feature/15 improve std out reporter (#19) * Replace usage of Set by List for SuiteResults to keep insert order. * Solved small problem in StdOutReporter to get correct numbers. * Add logic to create "SKIPPED" Test Results when Suite Before fails. --- project/Dependencies.scala | 2 +- .../absa/testing/scapi/ScAPIRunner.scala | 2 +- .../scapi/reporter/StdOutReporter.scala | 19 +++++++----- .../scapi/suite/runner/SuiteRunner.scala | 29 ++++++++++++------- .../scapi/reporter/StdOutReporterTest.scala | 6 ++-- .../scapi/suite/runner/SuiteRunnerTest.scala | 10 +++---- 6 files changed, 39 insertions(+), 29 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 7799ffb..5d99c25 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -42,7 +42,7 @@ object Dependencies { "org.apache.logging.log4j" % "log4j-api" % loggerVersion, // test - "org.scalameta" %% "munit" % munitVersion % Test + "org.scalameta" %% "munit" % munitVersion % Test ) } diff --git a/testApi/src/main/scala/africa/absa/testing/scapi/ScAPIRunner.scala b/testApi/src/main/scala/africa/absa/testing/scapi/ScAPIRunner.scala index 37cb092..faeef59 100644 --- a/testApi/src/main/scala/africa/absa/testing/scapi/ScAPIRunner.scala +++ b/testApi/src/main/scala/africa/absa/testing/scapi/ScAPIRunner.scala @@ -59,7 +59,7 @@ object ScAPIRunner { Logger.info("Validate only => end run.") } else { Logger.info("Running tests") - val testResults: Set[SuiteResults] = SuiteRunner.runSuites(suiteBundles, environment, () => new RestClient(ScAPIRequestSender)) + val testResults: List[SuiteResults] = SuiteRunner.runSuites(suiteBundles, environment, () => new RestClient(ScAPIRequestSender)) StdOutReporter.printReport(testResults) } } diff --git a/testApi/src/main/scala/africa/absa/testing/scapi/reporter/StdOutReporter.scala b/testApi/src/main/scala/africa/absa/testing/scapi/reporter/StdOutReporter.scala index 8f134dd..5dfde82 100644 --- a/testApi/src/main/scala/africa/absa/testing/scapi/reporter/StdOutReporter.scala +++ b/testApi/src/main/scala/africa/absa/testing/scapi/reporter/StdOutReporter.scala @@ -27,7 +27,7 @@ object StdOutReporter { * * @param testResults The set of test suite results to be reported. */ - def printReport(testResults: Set[SuiteResults]): Unit = { + def printReport(testResults: List[SuiteResults]): Unit = { def createFormattedLine(line: Option[String] = None, maxChars: Int = 80, repeatChar: Char = '*'): String = line match { case Some(text) => s"${repeatChar.toString * ((maxChars - text.length - 2) / 2)} $text ${repeatChar.toString * ((maxChars - text.length - 2) / 2)}" @@ -58,16 +58,19 @@ object StdOutReporter { printHeader("Simple Text Report") - val successCount = testResults.count(_.status == SuiteResults.Success) - val failureCount = testResults.size - successCount + val successCount = testResults.count(r => r.status == SuiteResults.Success && r.resultType == SuiteResults.RESULT_TYPE_TEST) + val failureCount = testResults.count(r => r.status == SuiteResults.Failure && r.resultType == SuiteResults.RESULT_TYPE_TEST) - println(s"Number of tests run: ${testResults.size}") + println(s"Number of tests run: ${successCount + failureCount}") println(s"Number of successful tests: $successCount") println(s"Number of failed tests: $failureCount") if (testResults.nonEmpty) { - val suiteSummary = testResults.groupBy(_.suiteName).map { - case (suiteName, results) => (suiteName, results.size, results.count(_.status == SuiteResults.Success)) + val suiteSummary = testResults + .filter(_.resultType == SuiteResults.RESULT_TYPE_TEST) + .groupBy(_.suiteName).map { + case (suiteName, results) => + (suiteName, results.size, results.count(_.status == SuiteResults.Success)) } printInnerHeader("Suites Summary") @@ -80,7 +83,7 @@ object StdOutReporter { printTableRowSplitter() println(s"| %-${maxSuiteLength}s | %-${maxTestLength}s | %-13s | %-7s | %-${maxTestCategoriesLength}s | ".format("Suite Name", "Test Name", "Duration (ms)", "Status", "Categories")) printTableRowSplitter() - val resultsList = testResults.toList.sortBy(_.suiteName) + val resultsList = testResults.filter(_.resultType == SuiteResults.RESULT_TYPE_TEST) resultsList.zipWithIndex.foreach { case (result, index) => val duration = result.duration.map(_.toString).getOrElse("NA") println(s"| %-${maxSuiteLength}s | %-${maxTestLength}s | %13s | %-7s | %-${maxTestCategoriesLength}s | ".format(result.suiteName, result.name, duration, result.status, result.categories.getOrElse(""))) @@ -91,7 +94,7 @@ object StdOutReporter { if (failureCount > 0) { printInnerHeader("Details of failed tests") - testResults.filter(_.status == SuiteResults.Failure).toList.sortBy(_.name).foreach { result => + testResults.filter(_.status == SuiteResults.Failure).sortBy(_.name).foreach { result => println(s"Suite: ${result.suiteName}") println(s"Test: ${result.name}") println(s"Error: ${result.errMessage.getOrElse("No details available")}") diff --git a/testApi/src/main/scala/africa/absa/testing/scapi/suite/runner/SuiteRunner.scala b/testApi/src/main/scala/africa/absa/testing/scapi/suite/runner/SuiteRunner.scala index 1806ef4..d9b2da1 100644 --- a/testApi/src/main/scala/africa/absa/testing/scapi/suite/runner/SuiteRunner.scala +++ b/testApi/src/main/scala/africa/absa/testing/scapi/suite/runner/SuiteRunner.scala @@ -37,30 +37,37 @@ object SuiteRunner { * @param environment The current environment. * @return Set of SuiteResults. */ - def runSuites(suiteBundles: Set[SuiteBundle], environment: Environment, restClientCreator: RestClientCreator): Set[SuiteResults] = { - suiteBundles.flatMap(suiteBundle => { + def runSuites(suiteBundles: Set[SuiteBundle], environment: Environment, restClientCreator: RestClientCreator): List[SuiteResults] = { + suiteBundles.foldLeft(List[SuiteResults]()) { (resultList, suiteBundle) => Logger.debug(s"Running Suite: ${suiteBundle.suite.endpoint}") - val resultSuiteBefore: Set[SuiteResults] = suiteBundle.suiteBefore.map { suiteBefore => + val resultSuiteBefore: List[SuiteResults] = suiteBundle.suiteBefore.toList.flatMap { suiteBefore => suiteBefore.methods.map { method => runSuiteBefore(suiteBundle.suite.endpoint, suiteBefore.name, method, environment, restClientCreator) } - }.getOrElse(Set.empty) + } - var resultSuite: Set[SuiteResults] = Set.empty - var resultSuiteAfter: Set[SuiteResults] = Set.empty + var resultSuite: List[SuiteResults] = List.empty + var resultSuiteAfter: List[SuiteResults] = List.empty if (!resultSuiteBefore.forall(_.isSuccess)) { Logger.error(s"Suite-Before for Suite: ${suiteBundle.suite.endpoint} has failed methods. Not executing main tests and Suite-After.") + resultSuite = resultSuite :+ SuiteResults.withBooleanStatus( + resultType = SuiteResults.RESULT_TYPE_TEST, + suiteName = suiteBundle.suite.endpoint, + name = "SKIPPED", + status = false, + duration = Some(0L), + categories = Some("SKIPPED")) } else { - resultSuite = suiteBundle.suite.tests.map(test => + resultSuite = suiteBundle.suite.tests.toList.map(test => this.runSuiteTest(suiteBundle.suite.endpoint, test, environment, restClientCreator)) - resultSuiteAfter = suiteBundle.suiteAfter.map { suiteAfter => + resultSuiteAfter = suiteBundle.suiteAfter.toList.flatMap { suiteAfter => suiteAfter.methods.map { method => runSuiteAfter(suiteBundle.suite.endpoint, suiteAfter.name, method, environment, restClientCreator) } - }.getOrElse(Set.empty) + } } RuntimeCache.expire(SuiteLevel) - resultSuiteBefore ++ resultSuite ++ resultSuiteAfter - }) + resultList ++ resultSuiteBefore ++ resultSuite ++ resultSuiteAfter + } } /** diff --git a/testApi/src/test/scala/africa/absa/testing/scapi/reporter/StdOutReporterTest.scala b/testApi/src/test/scala/africa/absa/testing/scapi/reporter/StdOutReporterTest.scala index 40345b5..e1b4018 100644 --- a/testApi/src/test/scala/africa/absa/testing/scapi/reporter/StdOutReporterTest.scala +++ b/testApi/src/test/scala/africa/absa/testing/scapi/reporter/StdOutReporterTest.scala @@ -23,7 +23,7 @@ import java.io.ByteArrayOutputStream class StdOutReporterTest extends FunSuite { - val successTestResults: Set[SuiteResults] = Set( + val successTestResults: List[SuiteResults] = List( SuiteResults.withBooleanStatus(SuiteResults.RESULT_TYPE_TEST, suiteName = "Suite 1", name = "Test 1", @@ -45,7 +45,7 @@ class StdOutReporterTest extends FunSuite { categories = Some("Category 3")) ) - val mixedSuccessTestResults: Set[SuiteResults] = Set( + val mixedSuccessTestResults: List[SuiteResults] = List( SuiteResults.withBooleanStatus(SuiteResults.RESULT_TYPE_TEST, suiteName = "Suite 1", name = "Test 1", @@ -74,7 +74,7 @@ class StdOutReporterTest extends FunSuite { val baos = new ByteArrayOutputStream() Console.withOut(baos) { - StdOutReporter.printReport(Set.empty) + StdOutReporter.printReport(List.empty) } // Get the output as a string diff --git a/testApi/src/test/scala/africa/absa/testing/scapi/suite/runner/SuiteRunnerTest.scala b/testApi/src/test/scala/africa/absa/testing/scapi/suite/runner/SuiteRunnerTest.scala index 8c3c8eb..b5a6239 100644 --- a/testApi/src/test/scala/africa/absa/testing/scapi/suite/runner/SuiteRunnerTest.scala +++ b/testApi/src/test/scala/africa/absa/testing/scapi/suite/runner/SuiteRunnerTest.scala @@ -101,7 +101,7 @@ class SuiteRunnerTest extends FunSuite { */ test("runSuite - SuiteBefore exists") { - val suiteResults: Set[SuiteResults] = SuiteRunner.runSuites(suitesBundles, environment, () => new RestClient(FakeScAPIRequestSender)) + val suiteResults: List[SuiteResults] = SuiteRunner.runSuites(suitesBundles, environment, () => new RestClient(FakeScAPIRequestSender)) val beforeSuiteResult: SuiteResults = suiteResults.find(result => result.resultType == SuiteResults.RESULT_TYPE_BEFORE_METHOD && result.suiteName == "endpoint2").get @@ -112,7 +112,7 @@ class SuiteRunnerTest extends FunSuite { } test("runSuite - SuiteBefore empty") { - val suiteResults: Set[SuiteResults] = SuiteRunner.runSuites(suitesBundleNoBefore, environment, () => new RestClient(FakeScAPIRequestSender)) + val suiteResults: List[SuiteResults] = SuiteRunner.runSuites(suitesBundleNoBefore, environment, () => new RestClient(FakeScAPIRequestSender)) val beforeSuiteResult: Option[SuiteResults] = suiteResults.find(result => result.resultType == SuiteResults.RESULT_TYPE_BEFORE_METHOD && result.suiteName == "endpoint2") @@ -122,7 +122,7 @@ class SuiteRunnerTest extends FunSuite { } test("runSuite - SuiteAfter exists") { - val suiteResults: Set[SuiteResults] = SuiteRunner.runSuites(suitesBundles, environment, () => new RestClient(FakeScAPIRequestSender)) + val suiteResults: List[SuiteResults] = SuiteRunner.runSuites(suitesBundles, environment, () => new RestClient(FakeScAPIRequestSender)) val afterSuiteResult: SuiteResults = suiteResults.find(result => result.resultType == SuiteResults.RESULT_TYPE_AFTER_METHOD && result.suiteName == "endpoint2").get @@ -133,7 +133,7 @@ class SuiteRunnerTest extends FunSuite { } test("runSuite - SuiteAfter empty") { - val suiteResults: Set[SuiteResults] = SuiteRunner.runSuites(suitesBundleNoAfter, environment, () => new RestClient(FakeScAPIRequestSender)) + val suiteResults: List[SuiteResults] = SuiteRunner.runSuites(suitesBundleNoAfter, environment, () => new RestClient(FakeScAPIRequestSender)) val afterSuiteResult: Option[SuiteResults] = suiteResults.find(result => result.resultType == SuiteResults.RESULT_TYPE_AFTER_METHOD && result.suiteName == "endpoint2") @@ -143,7 +143,7 @@ class SuiteRunnerTest extends FunSuite { } test("runSuite - SuiteAfter empty methods") { - val suiteResults: Set[SuiteResults] = SuiteRunner.runSuites(suitesBundleAfterMethodNotSupported, environment, () => new RestClient(FakeScAPIRequestSender)) + val suiteResults: List[SuiteResults] = SuiteRunner.runSuites(suitesBundleAfterMethodNotSupported, environment, () => new RestClient(FakeScAPIRequestSender)) val afterSuiteResult: SuiteResults = suiteResults.find(result => result.resultType == SuiteResults.RESULT_TYPE_AFTER_METHOD && result.suiteName == "endpoint2").get