Feature/15 improve std out reporter (#19)

* Replace usage of Set by List for SuiteResults to keep insert order.
* Solved small problem in StdOutReporter to get correct numbers.
* Add logic to create "SKIPPED" Test Results when Suite Before fails.

project/Dependencies.scala

@@ -42,7 +42,7 @@ object Dependencies {
     "org.apache.logging.log4j"  % "log4j-api"               % loggerVersion,
 
     // test
-    "org.scalameta"       %% "munit"                  % munitVersion          % Test
+    "org.scalameta"             %% "munit"                  % munitVersion          % Test
 
   )
 }

testApi/src/main/scala/africa/absa/testing/scapi/ScAPIRunner.scala

@@ -59,7 +59,7 @@ object ScAPIRunner {
       Logger.info("Validate only => end run.")
     } else {
       Logger.info("Running tests")
-      val testResults: Set[SuiteResults] = SuiteRunner.runSuites(suiteBundles, environment, () => new RestClient(ScAPIRequestSender))
+      val testResults: List[SuiteResults] = SuiteRunner.runSuites(suiteBundles, environment, () => new RestClient(ScAPIRequestSender))
       StdOutReporter.printReport(testResults)
     }
   }

testApi/src/main/scala/africa/absa/testing/scapi/reporter/StdOutReporter.scala

@@ -27,7 +27,7 @@ object StdOutReporter {
    *
    * @param testResults The set of test suite results to be reported.
    */
-  def printReport(testResults: Set[SuiteResults]): Unit = {
+  def printReport(testResults: List[SuiteResults]): Unit = {
     def createFormattedLine(line: Option[String] = None, maxChars: Int = 80, repeatChar: Char = '*'): String =
       line match {
         case Some(text) => s"${repeatChar.toString * ((maxChars - text.length - 2) / 2)} $text ${repeatChar.toString * ((maxChars - text.length - 2) / 2)}"
@@ -58,16 +58,19 @@ object StdOutReporter {
 
     printHeader("Simple Text Report")
 
-    val successCount = testResults.count(_.status == SuiteResults.Success)
-    val failureCount = testResults.size - successCount
+    val successCount = testResults.count(r => r.status == SuiteResults.Success && r.resultType == SuiteResults.RESULT_TYPE_TEST)
+    val failureCount = testResults.count(r => r.status == SuiteResults.Failure && r.resultType == SuiteResults.RESULT_TYPE_TEST)
 
-    println(s"Number of tests run: ${testResults.size}")
+    println(s"Number of tests run: ${successCount + failureCount}")
     println(s"Number of successful tests: $successCount")
     println(s"Number of failed tests: $failureCount")
 
     if (testResults.nonEmpty) {
-      val suiteSummary = testResults.groupBy(_.suiteName).map {
-        case (suiteName, results) => (suiteName, results.size, results.count(_.status == SuiteResults.Success))
+      val suiteSummary = testResults
+        .filter(_.resultType == SuiteResults.RESULT_TYPE_TEST)
+        .groupBy(_.suiteName).map {
+          case (suiteName, results) =>
+            (suiteName, results.size, results.count(_.status == SuiteResults.Success))
       }
 
       printInnerHeader("Suites Summary")
@@ -80,7 +83,7 @@ object StdOutReporter {
       printTableRowSplitter()
       println(s"| %-${maxSuiteLength}s | %-${maxTestLength}s | %-13s | %-7s | %-${maxTestCategoriesLength}s | ".format("Suite Name", "Test Name", "Duration (ms)", "Status", "Categories"))
       printTableRowSplitter()
-      val resultsList = testResults.toList.sortBy(_.suiteName)
+      val resultsList = testResults.filter(_.resultType == SuiteResults.RESULT_TYPE_TEST)
       resultsList.zipWithIndex.foreach { case (result, index) =>
         val duration = result.duration.map(_.toString).getOrElse("NA")
         println(s"| %-${maxSuiteLength}s | %-${maxTestLength}s | %13s | %-7s | %-${maxTestCategoriesLength}s | ".format(result.suiteName, result.name, duration, result.status, result.categories.getOrElse("")))
@@ -91,7 +94,7 @@ object StdOutReporter {
 
       if (failureCount > 0) {
         printInnerHeader("Details of failed tests")
-        testResults.filter(_.status == SuiteResults.Failure).toList.sortBy(_.name).foreach { result =>
+        testResults.filter(_.status == SuiteResults.Failure).sortBy(_.name).foreach { result =>
           println(s"Suite: ${result.suiteName}")
           println(s"Test: ${result.name}")
           println(s"Error: ${result.errMessage.getOrElse("No details available")}")

testApi/src/main/scala/africa/absa/testing/scapi/suite/runner/SuiteRunner.scala

@@ -37,30 +37,37 @@ object SuiteRunner {
    * @param environment  The current environment.
    * @return Set of SuiteResults.
    */
-  def runSuites(suiteBundles: Set[SuiteBundle], environment: Environment, restClientCreator: RestClientCreator): Set[SuiteResults] = {
-    suiteBundles.flatMap(suiteBundle => {
+  def runSuites(suiteBundles: Set[SuiteBundle], environment: Environment, restClientCreator: RestClientCreator): List[SuiteResults] = {
+    suiteBundles.foldLeft(List[SuiteResults]()) { (resultList, suiteBundle) =>
       Logger.debug(s"Running Suite: ${suiteBundle.suite.endpoint}")
 
-      val resultSuiteBefore: Set[SuiteResults] = suiteBundle.suiteBefore.map { suiteBefore =>
+      val resultSuiteBefore: List[SuiteResults] = suiteBundle.suiteBefore.toList.flatMap { suiteBefore =>
         suiteBefore.methods.map { method => runSuiteBefore(suiteBundle.suite.endpoint, suiteBefore.name, method, environment, restClientCreator) }
-      }.getOrElse(Set.empty)
+      }
 
-      var resultSuite: Set[SuiteResults] = Set.empty
-      var resultSuiteAfter: Set[SuiteResults] = Set.empty
+      var resultSuite: List[SuiteResults] = List.empty
+      var resultSuiteAfter: List[SuiteResults] = List.empty
       if (!resultSuiteBefore.forall(_.isSuccess)) {
         Logger.error(s"Suite-Before for Suite: ${suiteBundle.suite.endpoint} has failed methods. Not executing main tests and Suite-After.")
+        resultSuite = resultSuite :+ SuiteResults.withBooleanStatus(
+          resultType = SuiteResults.RESULT_TYPE_TEST,
+          suiteName = suiteBundle.suite.endpoint,
+          name = "SKIPPED",
+          status = false,
+          duration = Some(0L),
+          categories = Some("SKIPPED"))
       } else {
-        resultSuite = suiteBundle.suite.tests.map(test =>
+        resultSuite = suiteBundle.suite.tests.toList.map(test =>
           this.runSuiteTest(suiteBundle.suite.endpoint, test, environment, restClientCreator))
 
-        resultSuiteAfter = suiteBundle.suiteAfter.map { suiteAfter =>
+        resultSuiteAfter = suiteBundle.suiteAfter.toList.flatMap { suiteAfter =>
           suiteAfter.methods.map { method => runSuiteAfter(suiteBundle.suite.endpoint, suiteAfter.name, method, environment, restClientCreator) }
-        }.getOrElse(Set.empty)
+        }
       }
 
       RuntimeCache.expire(SuiteLevel)
-      resultSuiteBefore ++ resultSuite ++ resultSuiteAfter
-    })
+      resultList ++ resultSuiteBefore ++ resultSuite ++ resultSuiteAfter
+    }
   }
 
   /**

testApi/src/test/scala/africa/absa/testing/scapi/reporter/StdOutReporterTest.scala

@@ -23,7 +23,7 @@ import java.io.ByteArrayOutputStream
 
 class StdOutReporterTest extends FunSuite {
 
-  val successTestResults: Set[SuiteResults] = Set(
+  val successTestResults: List[SuiteResults] = List(
     SuiteResults.withBooleanStatus(SuiteResults.RESULT_TYPE_TEST,
       suiteName = "Suite 1",
       name = "Test 1",
@@ -45,7 +45,7 @@ class StdOutReporterTest extends FunSuite {
       categories = Some("Category 3"))
   )
 
-  val mixedSuccessTestResults: Set[SuiteResults] = Set(
+  val mixedSuccessTestResults: List[SuiteResults] = List(
     SuiteResults.withBooleanStatus(SuiteResults.RESULT_TYPE_TEST,
       suiteName = "Suite 1",
       name = "Test 1",
@@ -74,7 +74,7 @@ class StdOutReporterTest extends FunSuite {
     val baos = new ByteArrayOutputStream()
 
     Console.withOut(baos) {
-      StdOutReporter.printReport(Set.empty)
+      StdOutReporter.printReport(List.empty)
     }
 
     // Get the output as a string

testApi/src/test/scala/africa/absa/testing/scapi/suite/runner/SuiteRunnerTest.scala

@@ -101,7 +101,7 @@ class SuiteRunnerTest extends FunSuite {
    */
 
   test("runSuite - SuiteBefore exists") {
-    val suiteResults: Set[SuiteResults] = SuiteRunner.runSuites(suitesBundles, environment, () => new RestClient(FakeScAPIRequestSender))
+    val suiteResults: List[SuiteResults] = SuiteRunner.runSuites(suitesBundles, environment, () => new RestClient(FakeScAPIRequestSender))
 
     val beforeSuiteResult: SuiteResults = suiteResults.find(result =>
       result.resultType == SuiteResults.RESULT_TYPE_BEFORE_METHOD && result.suiteName == "endpoint2").get
@@ -112,7 +112,7 @@ class SuiteRunnerTest extends FunSuite {
   }
 
   test("runSuite - SuiteBefore empty") {
-    val suiteResults: Set[SuiteResults] = SuiteRunner.runSuites(suitesBundleNoBefore, environment, () => new RestClient(FakeScAPIRequestSender))
+    val suiteResults: List[SuiteResults] = SuiteRunner.runSuites(suitesBundleNoBefore, environment, () => new RestClient(FakeScAPIRequestSender))
 
     val beforeSuiteResult: Option[SuiteResults] = suiteResults.find(result =>
       result.resultType == SuiteResults.RESULT_TYPE_BEFORE_METHOD && result.suiteName == "endpoint2")
@@ -122,7 +122,7 @@ class SuiteRunnerTest extends FunSuite {
   }
 
   test("runSuite - SuiteAfter exists") {
-    val suiteResults: Set[SuiteResults] = SuiteRunner.runSuites(suitesBundles, environment, () => new RestClient(FakeScAPIRequestSender))
+    val suiteResults: List[SuiteResults] = SuiteRunner.runSuites(suitesBundles, environment, () => new RestClient(FakeScAPIRequestSender))
 
     val afterSuiteResult: SuiteResults = suiteResults.find(result =>
       result.resultType == SuiteResults.RESULT_TYPE_AFTER_METHOD && result.suiteName == "endpoint2").get
@@ -133,7 +133,7 @@ class SuiteRunnerTest extends FunSuite {
   }
 
   test("runSuite - SuiteAfter empty") {
-    val suiteResults: Set[SuiteResults] = SuiteRunner.runSuites(suitesBundleNoAfter, environment, () => new RestClient(FakeScAPIRequestSender))
+    val suiteResults: List[SuiteResults] = SuiteRunner.runSuites(suitesBundleNoAfter, environment, () => new RestClient(FakeScAPIRequestSender))
 
     val afterSuiteResult: Option[SuiteResults] = suiteResults.find(result =>
       result.resultType == SuiteResults.RESULT_TYPE_AFTER_METHOD && result.suiteName == "endpoint2")
@@ -143,7 +143,7 @@ class SuiteRunnerTest extends FunSuite {
   }
 
   test("runSuite - SuiteAfter empty methods") {
-    val suiteResults: Set[SuiteResults] = SuiteRunner.runSuites(suitesBundleAfterMethodNotSupported, environment, () => new RestClient(FakeScAPIRequestSender))
+    val suiteResults: List[SuiteResults] = SuiteRunner.runSuites(suitesBundleAfterMethodNotSupported, environment, () => new RestClient(FakeScAPIRequestSender))
 
     val afterSuiteResult: SuiteResults = suiteResults.find(result =>
       result.resultType == SuiteResults.RESULT_TYPE_AFTER_METHOD && result.suiteName == "endpoint2").get