Merge pull request #1514 from znsio/rand-resolve-failure-capture

Ensure random resolve failures appear in HTML report with example names.
znsio · Jan 2, 2025 · cabfaae · cabfaae
2 parents da5690e + 67bf04d
commit cabfaae
Show file tree

Hide file tree

Showing 6 changed files with 108 additions and 27 deletions.
diff --git a/core/src/main/kotlin/io/specmatic/core/Scenario.kt b/core/src/main/kotlin/io/specmatic/core/Scenario.kt
@@ -416,7 +416,10 @@ data class Scenario(
             attempt {
                 val newResponsePattern: HttpResponsePattern = this.httpResponsePattern.withResponseExampleValue(row, resolver)
 
-                val resolvedRow = ExampleProcessor.resolve(row, ExampleProcessor::defaultIfNotExits)
+                val resolvedRow = try { ExampleProcessor.resolve(row, ExampleProcessor::defaultIfNotExits) } catch (e: Throwable) {
+                    return@attempt sequenceOf(HasException<Scenario>(e, message = row.name, breadCrumb = ""))
+                }
+
                 val (newRequestPatterns: Sequence<ReturnValue<HttpRequestPattern>>, generativePrefix: String) = when (isNegative) {
                     false -> Pair(httpRequestPattern.newBasedOn(resolvedRow, resolver, httpResponsePattern.status), flagsBased.positivePrefix)
                     else -> Pair(httpRequestPattern.negativeBasedOn(resolvedRow, resolver.copy(isNegative = true)), flagsBased.negativePrefix)
@@ -437,7 +440,7 @@ data class Scenario(
                                 ), (newHttpRequestPattern as HasValue<HttpRequestPattern>).valueDetails
                             )
                         },
-                        orException = { e -> e.addDetails(message = row.name, breadCrumb = "").cast() },
+                        orException = { e -> e.copy(message = row.name).cast() },
                         orFailure = { f -> f.addDetails(message = row.name, breadCrumb = "").cast() }
                     )
                 }

diff --git a/core/src/main/kotlin/io/specmatic/test/ExamplePreProcessor.kt b/core/src/main/kotlin/io/specmatic/test/ExamplePreProcessor.kt
@@ -3,7 +3,6 @@ package io.specmatic.test
 import io.ktor.http.*
 import io.ktor.util.*
 import io.specmatic.core.*
-import io.specmatic.core.log.consoleLog
 import io.specmatic.core.pattern.*
 import io.specmatic.core.utilities.exceptionCauseMessage
 import io.specmatic.core.value.*
@@ -30,13 +29,11 @@ object ExampleProcessor {
 
         val configFile = File(configFilePath)
         if (!configFile.exists()) {
-            consoleLog("Could not find the CONFIG at path ${configFile.canonicalPath}")
-            return JSONObjectValue(emptyMap())
+            throw ContractException(breadCrumb = configFilePath, errorMessage = "Could not find the CONFIG at path ${configFile.canonicalPath}")
         }
 
         return runCatching { parsedJSONObject(configFile.readText()) }.getOrElse { e ->
-            consoleLog("Error loading CONFIG $configFilePath: ${exceptionCauseMessage(e)}")
-            JSONObjectValue(emptyMap())
+            throw ContractException(breadCrumb = configFilePath, errorMessage = "Could not parse the CONFIG at path ${configFile.canonicalPath}: ${exceptionCauseMessage(e)}")
         }.also {
             it.findFirstChildByPath("url")?.let {
                 url -> System.setProperty("testBaseURL", url.toStringLiteral())

diff --git a/core/src/main/kotlin/io/specmatic/test/ScenarioTestGenerationException.kt b/core/src/main/kotlin/io/specmatic/test/ScenarioTestGenerationException.kt
@@ -25,6 +25,7 @@ class ScenarioTestGenerationException(
     }
 
     private val httpRequest: HttpRequest = scenario.exampleRow?.requestExample ?: HttpRequest(path = scenario.path, method = scenario.method)
+    private val errorMessage: String = if (scenario.exampleRow != null && e is ContractException) "" else message
 
     override fun toScenarioMetadata() = scenario.toScenarioMetadata()
 
@@ -65,8 +66,8 @@ class ScenarioTestGenerationException(
 
     fun error(): Pair<Result, HttpResponse?> {
         val result: Result = when(e) {
-            is ContractException -> Result.Failure(message, e.failure(), breadCrumb = breadCrumb ?: "").updateScenario(scenario)
-            else -> Result.Failure(message + " - " + exceptionCauseMessage(e), breadCrumb = breadCrumb ?: "").updateScenario(scenario)
+            is ContractException -> Result.Failure(errorMessage, e.failure(), breadCrumb = breadCrumb ?: "").updateScenario(scenario)
+            else -> Result.Failure(errorMessage + " - " + exceptionCauseMessage(e), breadCrumb = breadCrumb ?: "").updateScenario(scenario)
         }
 
         return Pair(result, null)

diff --git a/core/src/test/kotlin/integration_tests/LoadTestsFromExternalisedFiles.kt b/core/src/test/kotlin/integration_tests/LoadTestsFromExternalisedFiles.kt
@@ -639,5 +639,37 @@ class LoadTestsFromExternalisedFiles {
                 """.trimIndent())
             }
         }
+
+        @Test
+        fun `should retain example information when rand resolve fails to find a substitution`() {
+            Flags.using(ADDITIONAL_EXAMPLE_PARAMS_FILE to "src/test/resources/openapi/config_and_entity_tests/incomplete_config.json") {
+                ExampleProcessor.cleanStores()
+                val feature = OpenApiSpecification.fromFile(
+                    "src/test/resources/openapi/config_and_entity_tests/spec.yaml"
+                ).toFeature().loadExternalisedExamples()
+
+                var requestsCount = 0
+                val results = feature.executeTests(object: TestExecutor {
+                    override fun execute(request: HttpRequest): HttpResponse {
+                        requestsCount += 1
+                        return HttpResponse(
+                            status = 201,
+                            body = parsedJSONObject("""{"id": 1}""").mergeWith(request.body)
+                        )
+                    }
+                }).results
+                val failure = results.filterIsInstance<Result.Failure>().first()
+                println(failure.scenario?.testDescription())
+                println(failure.reportString())
+
+                assertThat(requestsCount).isEqualTo(1)
+                assertThat(results).hasSize(2)
+                assertThat(failure.scenario?.testDescription()).containsIgnoringWhitespaces("Scenario: PATCH /pets/(id:number) -> 200 | EX:patch")
+                assertThat(failure.reportString()).containsIgnoringWhitespaces("""
+                >> CONFIG.patch.Pet.name
+                Couldn't pick a random value from "CONFIG.patch.Pet.name" that was not equal to "Tom"
+                """.trimIndent())
+            }
+        }
     }
 }
diff --git a/core/src/test/kotlin/io/specmatic/test/ExampleProcessorTest.kt b/core/src/test/kotlin/io/specmatic/test/ExampleProcessorTest.kt
@@ -6,15 +6,15 @@ import io.specmatic.core.NoBodyValue
 import io.specmatic.core.QueryParameters
 import io.specmatic.core.pattern.ContractException
 import io.specmatic.core.pattern.Row
+import io.specmatic.core.utilities.Flags
 import io.specmatic.core.utilities.Flags.Companion.ADDITIONAL_EXAMPLE_PARAMS_FILE
 import io.specmatic.core.value.JSONArrayValue
 import io.specmatic.core.value.JSONObjectValue
 import io.specmatic.core.value.NumberValue
 import io.specmatic.core.value.StringValue
 import io.specmatic.test.asserts.AssertComparisonTest.Companion.toFactStore
 import org.assertj.core.api.Assertions.assertThat
-import org.junit.jupiter.api.AfterAll
-import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Nested
 import org.junit.jupiter.api.Test
@@ -65,25 +65,21 @@ class ExampleProcessorTest {
                 )
             )
         )
-
-        @JvmStatic
-        @BeforeAll
-        fun setup(@TempDir tempDir: File) {
-            val configFile = File(tempDir, "config.json")
-            configFile.writeText(payloadConfig.toStringLiteral())
-            System.setProperty(ADDITIONAL_EXAMPLE_PARAMS_FILE, configFile.canonicalPath)
-        }
-
-        @JvmStatic
-        @AfterAll
-        fun cleanup() {
-            System.clearProperty(ADDITIONAL_EXAMPLE_PARAMS_FILE)
-            ExampleProcessor.cleanStores()
-        }
     }
 
     @BeforeEach
-    fun cleanStores() { ExampleProcessor.cleanStores() }
+    fun setupConfig(@TempDir tempDir: File) {
+        val configFile = File(tempDir, "config.json")
+        configFile.writeText(payloadConfig.toStringLiteral())
+        System.setProperty(ADDITIONAL_EXAMPLE_PARAMS_FILE, configFile.canonicalPath)
+        ExampleProcessor.cleanStores()
+    }
+
+    @AfterEach
+    fun cleanStores() {
+        System.clearProperty(ADDITIONAL_EXAMPLE_PARAMS_FILE)
+        ExampleProcessor.cleanStores()
+    }
 
     @Nested
     inner class DelayedRandomSubstitution {
@@ -376,4 +372,28 @@ class ExampleProcessorTest {
         Could not merge http response body with ENTITY for example "test"
         """.trimIndent())
     }
+
+    @Test
+    fun `should throw an exception when defined config is not found`() {
+        Flags.using(ADDITIONAL_EXAMPLE_PARAMS_FILE to "/does/not/exist") {
+            val exception = assertThrows<ContractException> { ExampleProcessor.cleanStores() }
+            assertThat(exception.report()).containsIgnoringWhitespaces("""
+            >> /does/not/exist 
+            Could not find the CONFIG at path ${File("/does/not/exist").canonicalPath}
+            """.trimIndent())
+        }
+    }
+
+    @Test
+    fun `should throw an exception when defined config is not valid`(@TempDir tempDir: File) {
+        val configFile = File(tempDir, "config.json")
+        configFile.writeText("10")
+        Flags.using(ADDITIONAL_EXAMPLE_PARAMS_FILE to configFile.canonicalPath) {
+            val exception = assertThrows<ContractException> { ExampleProcessor.cleanStores() }
+            assertThat(exception.report()).containsIgnoringWhitespaces("""
+            >> ${configFile.canonicalPath}  
+            Could not parse the CONFIG at path ${configFile.canonicalPath}: Expected json object, actual was 10
+            """.trimIndent())
+        }
+    }
 }
diff --git a/core/src/test/resources/openapi/config_and_entity_tests/incomplete_config.json b/core/src/test/resources/openapi/config_and_entity_tests/incomplete_config.json
@@ -0,0 +1,28 @@
+{
+  "post": {
+    "Pet": {
+      "name": "Tom",
+      "tag": ["cat"],
+      "details": {
+        "color": "black"
+      },
+      "adopted": true,
+      "age": 10,
+      "birthdate": "2025-01-01"
+    }
+  },
+  "patch": {
+    "Pet": {
+      "name": ["Tom"],
+      "tag": [
+        ["cat"]
+      ],
+      "details": [
+        { "color": "black" }
+      ],
+      "adopted": [true],
+      "age": [10],
+      "birthdate": ["2025-01-01"]
+    }
+  }
+}