From f0007c30e6821396b7f2db13c3d8b1a5e0b28856 Mon Sep 17 00:00:00 2001 From: slinkydeveloper Date: Fri, 2 Aug 2024 18:16:15 +0200 Subject: [PATCH] Emojis!!! --- README.md | 11 +- build.gradle.kts | 3 +- gradle/libs.versions.toml | 4 +- .../restate/sdktesting/infra/ServiceSpec.kt | 2 +- .../sdktesting/junit/ExecutionResult.kt | 199 ++++++++++++++++++ .../junit/ExecutionResultCollector.kt | 69 ++++++ .../dev/restate/sdktesting/junit/TestSuite.kt | 68 +++--- .../restate/sdktesting/junit/TestSuites.kt | 8 +- .../dev/restate/sdktesting/junit/utils.kt | 26 +++ .../kotlin/dev/restate/sdktesting/main.kt | 85 ++++---- .../sdktesting/tests/NonDeterminismErrors.kt | 5 +- 11 files changed, 405 insertions(+), 75 deletions(-) create mode 100644 src/main/kotlin/dev/restate/sdktesting/junit/ExecutionResult.kt create mode 100644 src/main/kotlin/dev/restate/sdktesting/junit/ExecutionResultCollector.kt create mode 100644 src/main/kotlin/dev/restate/sdktesting/junit/utils.kt diff --git a/README.md b/README.md index a160cbf..bd48005 100644 --- a/README.md +++ b/README.md @@ -12,4 +12,13 @@ TODO, more coming soon ## Local debugging usage * Run the service with your IDE and the debugger -* Run `java -jar restate-sdk-test-suite.jar debug --test-suite= --test-name= default-service=9080` \ No newline at end of file +* Run `java -jar restate-sdk-test-suite.jar debug --test-suite= --test-name= default-service=9080` + +## Releasing + +Just push a new git tag: + +```shell +git tag v1.1 +git push --tags +``` \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 4a4efe9..071791e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -22,7 +22,8 @@ repositories { } dependencies { - implementation("com.github.ajalt.clikt:clikt:4.2.2") + implementation(libs.clikt) + implementation(libs.mordant) implementation(libs.restate.admin) implementation(libs.restate.sdk.common) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b83f7ff..800ca59 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,4 +6,6 @@ ktor = "2.3.9" ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } ktor-client-java = { module = "io.ktor:ktor-client-java", version.ref = "ktor" } ktor-client-content-negotiation = { module = "io.ktor:ktor-client-content-negotiation", version.ref = "ktor" } -ktor-serialization-kotlinx-json = { module = "io.ktor:ktor-serialization-kotlinx-json", version.ref = "ktor" } \ No newline at end of file +ktor-serialization-kotlinx-json = { module = "io.ktor:ktor-serialization-kotlinx-json", version.ref = "ktor" } +mordant = { module = "com.github.ajalt.mordant:mordant", version = "2.7.2" } +clikt = { module = "com.github.ajalt.clikt:clikt", version = "4.2.2" } \ No newline at end of file diff --git a/src/main/kotlin/dev/restate/sdktesting/infra/ServiceSpec.kt b/src/main/kotlin/dev/restate/sdktesting/infra/ServiceSpec.kt index d55ce1f..9085695 100644 --- a/src/main/kotlin/dev/restate/sdktesting/infra/ServiceSpec.kt +++ b/src/main/kotlin/dev/restate/sdktesting/infra/ServiceSpec.kt @@ -69,7 +69,7 @@ data class ServiceSpec( } is LocalForwardServiceDeploymentConfig -> { Testcontainers.exposeHostPorts(serviceConfig.port) - println( + LOG.warn( """ Service spec '$name' won't deploy a container, but will use locally running service deployment: * Should be available at 'localhost:${serviceConfig.port}' diff --git a/src/main/kotlin/dev/restate/sdktesting/junit/ExecutionResult.kt b/src/main/kotlin/dev/restate/sdktesting/junit/ExecutionResult.kt new file mode 100644 index 0000000..455b0ca --- /dev/null +++ b/src/main/kotlin/dev/restate/sdktesting/junit/ExecutionResult.kt @@ -0,0 +1,199 @@ +// Copyright (c) 2023 - Restate Software, Inc., Restate GmbH +// +// This file is part of the Restate SDK Test suite tool, +// which is released under the MIT license. +// +// You can find a copy of the license in file LICENSE in the root +// directory of this repository or package, or at +// https://github.com/restatedev/sdk-test-suite/blob/main/LICENSE +package dev.restate.sdktesting.junit + +import com.github.ajalt.mordant.rendering.TextColors.green +import com.github.ajalt.mordant.rendering.TextColors.red +import com.github.ajalt.mordant.rendering.TextStyles.bold +import com.github.ajalt.mordant.terminal.Terminal +import java.io.PrintWriter +import java.lang.IllegalStateException +import java.util.* +import kotlin.math.min +import kotlin.time.TimeSource +import org.junit.platform.launcher.TestIdentifier +import org.junit.platform.launcher.TestPlan + +class ExecutionResult( + val testSuite: String, + private val testPlan: TestPlan, + private val classesResults: Map, + private val testResults: Map, + timeStarted: TimeSource.Monotonic.ValueTimeMark, + timeFinished: TimeSource.Monotonic.ValueTimeMark +) { + + val succeededTests = testResults.values.count { it is Succeeded } + val executedTests = testResults.size + val succeededClasses = classesResults.values.count { it is Succeeded } + val executedClasses = classesResults.size + val executionDuration = timeFinished - timeStarted + + companion object { + private const val TAB = " " + private const val DOUBLE_TAB = TAB + TAB + private const val DEFAULT_MAX_STACKTRACE_LINES = 15 + + private const val CAUSED_BY = "Caused by: " + private const val SUPPRESSED = "Suppressed: " + private const val CIRCULAR = "Circular reference: " + } + + sealed interface TestResult + + data object Succeeded : TestResult + + data object Aborted : TestResult + + data class Failed(val throwable: Throwable?) : TestResult + + val failedTests: List + get() { + return classesResults + .toList() + .filter { it.second is Failed || it.second is Aborted } + .map { it.first } + + testResults + .toList() + .filter { it.second is Failed || it.second is Aborted } + .map { it.first } + } + + fun printShortSummary(terminal: Terminal) { + // Compute test counters + val testsStyle = if (succeededTests == testResults.size) green else red + val testsInfoLine = testsStyle("""* Succeeded tests: $succeededTests / ${executedTests}""") + + // Compute classes counters + val failedClasses = executedClasses - succeededClasses + val classesStyle = if (failedClasses != 0) red else green + val classesInfoLine = classesStyle("""* Failed classes initialization: $failedClasses""") + + // Terminal print + terminal.println( + """ + ${bold("==== $testSuite results")} + $testsInfoLine + $classesInfoLine + * Execution time: $executionDuration + """ + .trimIndent()) + } + + fun printFailuresTo(terminal: Terminal, maxStackTraceLines: Int = DEFAULT_MAX_STACKTRACE_LINES) { + + val classesFailures = + this.classesResults.toList().filter { it.second is Aborted || it.second is Failed } + val testsFailures = + this.classesResults.toList().filter { it.second is Aborted || it.second is Failed } + + fun printFailures(failureList: List>) { + for (failure in failureList) { + terminal.println("$TAB${describeTestIdentifier(testSuite, testPlan, failure.first)}") + describeTestIdentifierSource(terminal, failure.first) + when (failure.second) { + Aborted -> terminal.println("${DOUBLE_TAB}=> ABORTED") + is Failed -> { + val throwable = (failure.second as Failed).throwable + if (throwable == null) { + terminal.println("${DOUBLE_TAB}=> UNKNOWN FAILURE") + } else { + terminal.println("$DOUBLE_TAB=> $throwable") + printStackTrace(PrintWriter(System.out), throwable, maxStackTraceLines) + } + } + Succeeded -> throw IllegalStateException() + } + } + } + + if (classesFailures.isEmpty() && testsFailures.isEmpty()) { + return + } + + terminal.println((red + bold)("== '$testSuite' FAILURES")) + + if (classesFailures.isNotEmpty()) { + terminal.println("Classes initialization failures ${red(classesFailures.size.toString())}:") + printFailures(classesFailures) + } + + if (testsFailures.isNotEmpty()) { + terminal.println("Test failures ${red(testsFailures.size.toString())}:") + printFailures(testsFailures) + } + } + + private fun describeTestIdentifierSource(terminal: Terminal, testIdentifier: TestIdentifier) { + testIdentifier.source.ifPresent { terminal.println("${DOUBLE_TAB}$it") } + } + + private fun printStackTrace(writer: PrintWriter, throwable: Throwable, max: Int) { + var max = max + if (throwable.cause != null || + (throwable.suppressed != null && throwable.suppressed.size > 0)) { + max = max / 2 + } + printStackTrace(writer, arrayOf(), throwable, "", DOUBLE_TAB + " ", HashSet(), max) + writer.flush() + } + + private fun printStackTrace( + writer: PrintWriter, + parentTrace: Array?, + throwable: Throwable?, + caption: String, + indentation: String, + seenThrowables: MutableSet, + max: Int + ) { + if (seenThrowables.contains(throwable)) { + writer.printf("%s%s[%s%s]%n", indentation, TAB, CIRCULAR, throwable) + return + } + seenThrowables.add(throwable) + + val trace = throwable!!.stackTrace + if (parentTrace != null && parentTrace.size > 0) { + writer.printf("%s%s%s%n", indentation, caption, throwable) + } + val duplicates = numberOfCommonFrames(trace, parentTrace) + val numDistinctFrames = trace.size - duplicates + val numDisplayLines = min(numDistinctFrames.toDouble(), max.toDouble()).toInt() + for (i in 0 until numDisplayLines) { + writer.printf("%s%s%s%n", indentation, TAB, trace[i]) + } + if (trace.size > max || duplicates != 0) { + writer.printf("%s%s%s%n", indentation, TAB, "[...]") + } + + for (suppressed in throwable.suppressed) { + printStackTrace(writer, trace, suppressed, SUPPRESSED, indentation + TAB, seenThrowables, max) + } + if (throwable.cause != null) { + printStackTrace(writer, trace, throwable.cause, CAUSED_BY, indentation, seenThrowables, max) + } + } + + private fun numberOfCommonFrames( + currentTrace: Array, + parentTrace: Array? + ): Int { + var currentIndex = currentTrace.size - 1 + var parentIndex = parentTrace!!.size - 1 + while (currentIndex >= 0 && parentIndex >= 0) { + if (currentTrace[currentIndex] != parentTrace[parentIndex]) { + break + } + currentIndex-- + parentIndex-- + } + return currentTrace.size - 1 - currentIndex + } +} diff --git a/src/main/kotlin/dev/restate/sdktesting/junit/ExecutionResultCollector.kt b/src/main/kotlin/dev/restate/sdktesting/junit/ExecutionResultCollector.kt new file mode 100644 index 0000000..0c569e3 --- /dev/null +++ b/src/main/kotlin/dev/restate/sdktesting/junit/ExecutionResultCollector.kt @@ -0,0 +1,69 @@ +// Copyright (c) 2023 - Restate Software, Inc., Restate GmbH +// +// This file is part of the Restate SDK Test suite tool, +// which is released under the MIT license. +// +// You can find a copy of the license in file LICENSE in the root +// directory of this repository or package, or at +// https://github.com/restatedev/sdk-test-suite/blob/main/LICENSE +package dev.restate.sdktesting.junit + +import dev.restate.sdktesting.junit.ExecutionResult.TestResult +import java.util.concurrent.ConcurrentHashMap +import kotlin.jvm.optionals.getOrNull +import kotlin.time.TimeSource +import org.junit.platform.engine.TestExecutionResult +import org.junit.platform.engine.support.descriptor.ClassSource +import org.junit.platform.engine.support.descriptor.MethodSource +import org.junit.platform.launcher.TestExecutionListener +import org.junit.platform.launcher.TestIdentifier +import org.junit.platform.launcher.TestPlan + +class ExecutionResultCollector(private val testSuite: String) : TestExecutionListener { + private var testPlan: TestPlan? = null + private val classesResults: ConcurrentHashMap = ConcurrentHashMap() + private val testResults: ConcurrentHashMap = ConcurrentHashMap() + + @Volatile private var timeStarted: TimeSource.Monotonic.ValueTimeMark? = null + @Volatile private var timeFinished: TimeSource.Monotonic.ValueTimeMark? = null + + /** Get the summary generated by this listener. */ + val results: ExecutionResult + get() { + return ExecutionResult( + testSuite, + testPlan!!, + classesResults.toMap(), + testResults.toMap(), + timeStarted!!, + timeFinished!!) + } + + override fun testPlanExecutionStarted(testPlan: TestPlan) { + this.testPlan = testPlan + this.timeStarted = TimeSource.Monotonic.markNow() + } + + override fun testPlanExecutionFinished(testPlan: TestPlan) { + this.timeFinished = TimeSource.Monotonic.markNow() + } + + override fun executionFinished( + testIdentifier: TestIdentifier, + testExecutionResult: TestExecutionResult + ) { + if (testIdentifier.source.getOrNull() is MethodSource && testIdentifier.isTest) { + testResults[testIdentifier] = testExecutionResult.toTestResult() + } else if (testIdentifier.source.getOrNull() is ClassSource) { + classesResults[testIdentifier] = testExecutionResult.toTestResult() + } + } + + private fun TestExecutionResult.toTestResult(): TestResult { + return when (this.status!!) { + TestExecutionResult.Status.SUCCESSFUL -> ExecutionResult.Succeeded + TestExecutionResult.Status.ABORTED -> ExecutionResult.Aborted + TestExecutionResult.Status.FAILED -> ExecutionResult.Failed(this.throwable.getOrNull()) + } + } +} diff --git a/src/main/kotlin/dev/restate/sdktesting/junit/TestSuite.kt b/src/main/kotlin/dev/restate/sdktesting/junit/TestSuite.kt index 50fde06..e7b90e5 100644 --- a/src/main/kotlin/dev/restate/sdktesting/junit/TestSuite.kt +++ b/src/main/kotlin/dev/restate/sdktesting/junit/TestSuite.kt @@ -8,13 +8,14 @@ // https://github.com/restatedev/sdk-test-suite/blob/main/LICENSE package dev.restate.sdktesting.junit +import com.github.ajalt.mordant.rendering.TextStyles.bold +import com.github.ajalt.mordant.terminal.Terminal import dev.restate.sdktesting.infra.BaseRestateDeployerExtension import dev.restate.sdktesting.infra.getGlobalConfig import dev.restate.sdktesting.infra.registerGlobalConfig import java.io.PrintWriter import java.nio.file.Path import kotlin.jvm.optionals.getOrNull -import kotlin.time.Duration.Companion.milliseconds import org.apache.logging.log4j.Level import org.apache.logging.log4j.ThreadContext import org.apache.logging.log4j.core.config.Configurator @@ -25,14 +26,9 @@ import org.junit.platform.engine.TestExecutionResult import org.junit.platform.engine.discovery.DiscoverySelectors import org.junit.platform.engine.support.descriptor.ClassSource import org.junit.platform.engine.support.descriptor.MethodSource -import org.junit.platform.launcher.LauncherConstants -import org.junit.platform.launcher.TagFilter -import org.junit.platform.launcher.TestExecutionListener -import org.junit.platform.launcher.TestIdentifier +import org.junit.platform.launcher.* import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder import org.junit.platform.launcher.core.LauncherFactory -import org.junit.platform.launcher.listeners.SummaryGeneratingListener -import org.junit.platform.launcher.listeners.TestExecutionSummary import org.junit.platform.reporting.legacy.xml.LegacyXmlReportGeneratingListener class TestSuite( @@ -41,18 +37,18 @@ class TestSuite( val junitIncludeTags: String ) { fun runTests( + terminal: Terminal, baseReportDir: Path, filters: List>, printToStdout: Boolean - ): TestExecutionSummary { + ): ExecutionResult { val reportDir = baseReportDir.resolve(name) - - println( - """ - ========================= $name ========================= - Report directory: $reportDir + terminal.println( """ - .trimIndent()) + |==== ${bold(name)} + |🗈 Report directory: $reportDir + """ + .trimMargin()) // Apply additional runtime envs registerGlobalConfig(getGlobalConfig().copy(additionalRuntimeEnvs = additionalEnvs)) @@ -76,8 +72,7 @@ class TestSuite( // Configure listeners val errWriter = PrintWriter(System.err) - // TODO replace this with our own listener - val summaryListener = SummaryGeneratingListener() + val executionResultCollector = ExecutionResultCollector(name) // TODO replace this with our own xml writer val xmlReportListener = LegacyXmlReportGeneratingListener(reportDir, errWriter) val redirectStdoutAndStderrListener = @@ -85,6 +80,30 @@ class TestSuite( reportDir.resolve("testrunner.stdout"), reportDir.resolve("testrunner.stderr"), errWriter) + val logTestEventsListener = + object : TestExecutionListener { + @Volatile var testPlan: TestPlan? = null + + override fun testPlanExecutionStarted(testPlan: TestPlan) { + this.testPlan = testPlan + } + + override fun executionFinished( + testIdentifier: TestIdentifier, + testExecutionResult: TestExecutionResult + ) { + if (testIdentifier.isTest) { + val name = describeTestIdentifier(name, testPlan!!, testIdentifier) + when (testExecutionResult.status!!) { + TestExecutionResult.Status.SUCCESSFUL -> terminal.println("✅ $name") + TestExecutionResult.Status.ABORTED -> terminal.println("❌ $name") + TestExecutionResult.Status.FAILED -> { + terminal.println("❌ $name") + } + } + } + } + } val injectLoggingContextListener = object : TestExecutionListener { val TEST_NAME = "test" @@ -113,24 +132,17 @@ class TestSuite( LauncherFactory.openSession().use { session -> val launcher = session.launcher launcher.registerTestExecutionListeners( - summaryListener, + executionResultCollector, + logTestEventsListener, xmlReportListener, redirectStdoutAndStderrListener, injectLoggingContextListener) launcher.execute(request) } - val report = summaryListener.summary!! + val report = executionResultCollector.results - println( - """ - * Succeeded tests: ${report.testsSucceededCount} / ${report.testsStartedCount} - * Succeeded test classes: ${report.containersSucceededCount - 1} / ${report.containersStartedCount - 1} - * Execution time: ${report.timeFinished.milliseconds - report.timeStarted.milliseconds} - """ - .trimIndent()) - val printWriter = PrintWriter(System.out) - report.printFailuresTo(printWriter) + report.printShortSummary(terminal) return report } @@ -139,7 +151,7 @@ class TestSuite( val builder = ConfigurationBuilderFactory.newConfigurationBuilder() val layout = builder.newLayout("PatternLayout") - layout.addAttribute("pattern", "%-4r [%t]%X %-5p %c - %m%n") + layout.addAttribute("pattern", "%-4r %-5p [%t]%notEmpty{[%X{test}]} %c{1.2.*} - %m%n") val fileAppender = builder.newAppender("log", "File") fileAppender.addAttribute("fileName", reportDir.resolve("testrunner.log").toString()) diff --git a/src/main/kotlin/dev/restate/sdktesting/junit/TestSuites.kt b/src/main/kotlin/dev/restate/sdktesting/junit/TestSuites.kt index 295a01f..a218e69 100644 --- a/src/main/kotlin/dev/restate/sdktesting/junit/TestSuites.kt +++ b/src/main/kotlin/dev/restate/sdktesting/junit/TestSuites.kt @@ -10,12 +10,12 @@ package dev.restate.sdktesting.junit object TestSuites { val DEFAULT_SUITE = TestSuite("default", emptyMap(), "none() | always-suspending") - val ALWAYS_SUSPENDING_SUITE = + private val ALWAYS_SUSPENDING_SUITE = TestSuite( "alwaysSuspending", mapOf("RESTATE_WORKER__INVOKER__INACTIVITY_TIMEOUT" to "0s"), "always-suspending | only-always-suspending") - val SINGLE_THREAD_SINGLE_PARTITION_SUITE = + private val SINGLE_THREAD_SINGLE_PARTITION_SUITE = TestSuite( "singleThreadSinglePartition", mapOf( @@ -23,14 +23,14 @@ object TestSuites { "RESTATE_DEFAULT_THREAD_POOL_SIZE" to "1", ), "none() | always-suspending") - val LAZY_STATE_SUITE = + private val LAZY_STATE_SUITE = TestSuite( "lazyState", mapOf( "RESTATE_WORKER__INVOKER__DISABLE_EAGER_STATE" to "true", ), "lazy-state") - val PERSISTED_TIMERS_SUITE = + private val PERSISTED_TIMERS_SUITE = TestSuite( "persistedTimers", mapOf("RESTATE_WORKER__NUM_TIMERS_IN_MEMORY_LIMIT" to "1"), "timers") diff --git a/src/main/kotlin/dev/restate/sdktesting/junit/utils.kt b/src/main/kotlin/dev/restate/sdktesting/junit/utils.kt new file mode 100644 index 0000000..98d7f70 --- /dev/null +++ b/src/main/kotlin/dev/restate/sdktesting/junit/utils.kt @@ -0,0 +1,26 @@ +// Copyright (c) 2023 - Restate Software, Inc., Restate GmbH +// +// This file is part of the Restate SDK Test suite tool, +// which is released under the MIT license. +// +// You can find a copy of the license in file LICENSE in the root +// directory of this repository or package, or at +// https://github.com/restatedev/sdk-test-suite/blob/main/LICENSE +package dev.restate.sdktesting.junit + +import kotlin.jvm.optionals.getOrNull +import org.junit.platform.launcher.TestIdentifier +import org.junit.platform.launcher.TestPlan + +fun describeTestIdentifier( + testSuite: String, + testPlan: TestPlan, + identifier: TestIdentifier? +): String { + if (identifier == null || identifier.parentId.isEmpty) { + return testSuite + } + val parent = + describeTestIdentifier(testSuite, testPlan, testPlan.getParent(identifier).getOrNull()) + return "$parent => ${identifier.displayName}" +} diff --git a/src/main/kotlin/dev/restate/sdktesting/main.kt b/src/main/kotlin/dev/restate/sdktesting/main.kt index a29e1a9..14b56a0 100644 --- a/src/main/kotlin/dev/restate/sdktesting/main.kt +++ b/src/main/kotlin/dev/restate/sdktesting/main.kt @@ -22,7 +22,12 @@ import com.github.ajalt.clikt.parameters.groups.provideDelegate import com.github.ajalt.clikt.parameters.options.* import com.github.ajalt.clikt.parameters.types.enum import com.github.ajalt.clikt.parameters.types.path +import com.github.ajalt.mordant.rendering.TextColors.green +import com.github.ajalt.mordant.rendering.TextColors.red +import com.github.ajalt.mordant.rendering.TextStyles.bold +import com.github.ajalt.mordant.terminal.Terminal import dev.restate.sdktesting.infra.* +import dev.restate.sdktesting.junit.ExecutionResult import dev.restate.sdktesting.junit.TestSuites import java.io.File import java.io.FileInputStream @@ -33,7 +38,6 @@ import java.time.format.DateTimeFormatter import kotlin.jvm.optionals.getOrNull import kotlin.system.exitProcess import kotlin.time.Duration -import kotlin.time.Duration.Companion.milliseconds import kotlinx.serialization.Serializable import org.junit.platform.engine.Filter import org.junit.platform.engine.discovery.ClassNameFilter @@ -109,6 +113,8 @@ Run test suite, executing the service as container. val imageName by argument() override fun run() { + val terminal = Terminal() + val restateDeployerConfig = RestateDeployerConfig( mapOf(ServiceSpec.DEFAULT_SERVICE_NAME to ContainerServiceDeploymentConfig(imageName))) @@ -127,16 +133,8 @@ Run test suite, executing the service as container. ExclusionsFile() } - data class AggregateResults( - var succededTests: Long = 0, - var startedTests: Long = 0, - var succeededClasses: Long = 0, - var startedClasses: Long = 0, - var totalDuration: Duration = Duration.ZERO - ) - - val aggregateResults = AggregateResults() - val failedTests = mutableMapOf>() + val reports = mutableListOf() + val newExclusions = mutableMapOf>() var newFailures = false for (testSuite in testSuites) { val exclusions = loadedExclusions.exclusions[testSuite.name] ?: emptyList() @@ -147,21 +145,14 @@ Run test suite, executing the service as container. val report = testSuite.runTests( - testRunnerOptions.reportDir, exclusionsFilters + cliOptionFilter, false) - - aggregateResults.succededTests += report.testsSucceededCount - aggregateResults.startedTests += report.testsStartedCount - aggregateResults.succeededClasses += - report.containersSucceededCount - 1 // Package is a test container - aggregateResults.startedClasses += - report.containersStartedCount - 1 // Package is a test container - aggregateResults.totalDuration += - report.timeFinished.milliseconds - report.timeStarted.milliseconds - - if (report.failures.isNotEmpty() || exclusions.isNotEmpty()) { - failedTests[testSuite.name] = - report.failures - .mapNotNull { it.testIdentifier.source.getOrNull() } + terminal, testRunnerOptions.reportDir, exclusionsFilters + cliOptionFilter, false) + + reports.add(report) + val failures = report.failedTests + if (failures.isNotEmpty() || exclusions.isNotEmpty()) { + newExclusions[testSuite.name] = + failures + .mapNotNull { it.source.getOrNull() } .mapNotNull { when (it) { is ClassSource -> it.className!! @@ -171,26 +162,43 @@ Run test suite, executing the service as container. } .distinct() + exclusions } - if (report.failures.isNotEmpty()) { + if (failures.isNotEmpty()) { newFailures = true } } + // Write out the exclusions file FileOutputStream(testRunnerOptions.reportDir.resolve("exclusions.new.yaml").toFile()).use { - Yaml.default.encodeToStream(ExclusionsFile(failedTests), it) + Yaml.default.encodeToStream(ExclusionsFile(newExclusions), it) } + // Print final report + val succeededTests = reports.sumOf { it.succeededTests } + val executedTests = reports.sumOf { it.executedTests } + val testsStyle = if (succeededTests == executedTests) green else red + val testsInfoLine = testsStyle("""* Succeeded tests: $succeededTests / ${executedTests}""") + + val failedClasses = reports.sumOf { it.executedClasses - it.succeededClasses } + val classesStyle = if (failedClasses != 0) red else green + val classesInfoLine = classesStyle("""* Failed classes initialization: $failedClasses""") + + val totalDuration = reports.fold(Duration.ZERO) { d, res -> d + res.executionDuration } + println( """ - ========================= Final results ========================= - All reports are available under: ${testRunnerOptions.reportDir} - - * Succeeded tests: ${aggregateResults.succededTests} / ${aggregateResults.startedTests} - * Succeeded test classes: ${aggregateResults.succeededClasses} / ${aggregateResults.startedClasses} - * Execution time: ${aggregateResults.totalDuration} + ${bold("========================= Final results =========================")} + 🗈 Report directory: ${testRunnerOptions.reportDir} + * Run test suites: ${reports.map { it.testSuite }} + $testsInfoLine + $classesInfoLine + * Execution time: $totalDuration """ .trimIndent()) + for (report in reports) { + report.printFailuresTo(terminal) + } + if (newFailures) { // Exit exitProcess(1) @@ -223,6 +231,8 @@ Run test suite, without executing the service inside a container. "Local containers name=ports. Example: '9080' (for default-service container), 'otherContainer=9081'") override fun run() { + val terminal = Terminal() + // Register global config of the deployer val restateDeployerConfig = RestateDeployerConfig( @@ -235,8 +245,11 @@ Run test suite, without executing the service inside a container. val testSuite = TestSuites.resolveSuites(testSuite)[0] val testFilters = listOf(ClassNameFilter.includeClassNamePatterns(testName)) - val report = testSuite.runTests(testRunnerOptions.reportDir, testFilters, true) - if (report.testsFailedCount != 0L) { + val report = testSuite.runTests(terminal, testRunnerOptions.reportDir, testFilters, true) + + report.printFailuresTo(terminal) + + if (report.failedTests.isNotEmpty()) { // Exit exitProcess(1) } diff --git a/src/main/kotlin/dev/restate/sdktesting/tests/NonDeterminismErrors.kt b/src/main/kotlin/dev/restate/sdktesting/tests/NonDeterminismErrors.kt index b98e6ac..446a1e4 100644 --- a/src/main/kotlin/dev/restate/sdktesting/tests/NonDeterminismErrors.kt +++ b/src/main/kotlin/dev/restate/sdktesting/tests/NonDeterminismErrors.kt @@ -19,8 +19,7 @@ import dev.restate.sdktesting.infra.RestateDeployer import dev.restate.sdktesting.infra.RestateDeployerExtension import dev.restate.sdktesting.infra.ServiceSpec import kotlinx.coroutines.test.runTest -import org.assertj.core.api.Assertions -import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.* import org.junit.jupiter.api.Tag import org.junit.jupiter.api.extension.RegisterExtension import org.junit.jupiter.api.parallel.Execution @@ -52,7 +51,7 @@ class NonDeterminismErrors { "setDifferentKey"]) @Execution(ExecutionMode.CONCURRENT) fun method(handlerName: String, @InjectClient ingressClient: Client) = runTest { - Assertions.assertThatThrownBy { + assertThatThrownBy { ingressClient.call( Target.virtualObject( NonDeterministicDefinitions.SERVICE_NAME, handlerName, handlerName),