diff --git a/core/src/main/kotlin/in/specmatic/conversions/EnvironmentAndPropertiesConfiguration.kt b/core/src/main/kotlin/in/specmatic/conversions/EnvironmentAndPropertiesConfiguration.kt index 7296c8aed..7eebe3c51 100644 --- a/core/src/main/kotlin/in/specmatic/conversions/EnvironmentAndPropertiesConfiguration.kt +++ b/core/src/main/kotlin/in/specmatic/conversions/EnvironmentAndPropertiesConfiguration.kt @@ -6,7 +6,6 @@ data class EnvironmentAndPropertiesConfiguration(val environmentVariables: Map - Results(results = results.results.plus(executeTest(scenario, httpClient)).toMutableList()) + feature.generateContractTests(emptyList()).fold(Results()) { results, contractTest -> + Results(results = results.results.plus(contractTest.runTest(httpClient).first)) } } } diff --git a/core/src/main/kotlin/in/specmatic/core/Feature.kt b/core/src/main/kotlin/in/specmatic/core/Feature.kt index 36e76506a..002ee3c65 100644 --- a/core/src/main/kotlin/in/specmatic/core/Feature.kt +++ b/core/src/main/kotlin/in/specmatic/core/Feature.kt @@ -189,16 +189,22 @@ data class Feature( } fun executeTests( - testExecutorFn: TestExecutor, + testExecutor: TestExecutor, suggestions: List = emptyList(), - scenarioNames: List = emptyList() - ): Results = - generateContractTestScenarios(suggestions) - .map { it.second.value } - .filter { scenarioNames.isEmpty() || scenarioNames.contains(it.name) } - .fold(Results()) { results, scenario -> - Results(results = results.results.plus(executeTest(scenario, testExecutorFn, flagsBased))) + testDescriptionFilter: List = emptyList() + ): Results { + return generateContractTests(suggestions) + .filter { contractTest -> + testDescriptionFilter.isEmpty() || + testDescriptionFilter.any { scenarioName -> + contractTest.testDescription().contains(scenarioName) + } + } + .fold(Results()) { results, contractTest -> + val (result, _) = contractTest.runTest(testExecutor) + Results(results = results.results.plus(result)) } + } fun setServerState(serverState: Map) { this.serverState = this.serverState.plus(serverState) @@ -300,7 +306,7 @@ data class Feature( return generateContractTestScenarios(suggestions).map { (originalScenario, returnValue) -> returnValue.realise( hasValue = { concreteTestScenario, comment -> - ScenarioTest( + ScenarioAsTest( concreteTestScenario, flagsBased, concreteTestScenario.sourceProvider, @@ -367,7 +373,11 @@ data class Feature( negativeScenarioResult.ifHasValue { result: HasValue -> val description = result.valueDetails.singleLineDescription() - HasValue(result.value.copy(descriptionFromPlugin = "${result.value.apiDescription} [${description}]")) + val tag = if(description.isNotBlank()) + " [${description}]" + else + "" + HasValue(result.value.copy(descriptionFromPlugin = "${result.value.apiDescription}$tag")) } } diff --git a/core/src/main/kotlin/in/specmatic/core/HttpResponsePattern.kt b/core/src/main/kotlin/in/specmatic/core/HttpResponsePattern.kt index ba45edc41..0b43ea51f 100644 --- a/core/src/main/kotlin/in/specmatic/core/HttpResponsePattern.kt +++ b/core/src/main/kotlin/in/specmatic/core/HttpResponsePattern.kt @@ -93,12 +93,7 @@ data class HttpResponsePattern( val body = response.body return when (response.status) { - status -> { - if(Flags.customResponse() && response.status.toString().startsWith("2") && body is JSONObjectValue && body.findFirstChildByPath("resultStatus.status")?.toStringLiteral() == "FAILED") - MatchFailure(mismatchResult("status $status and resultStatus.status == \"SUCCESS\"", "status ${response.status} and resultStatus.status == \"${body.findFirstChildByPath("resultStatus.status")?.toStringLiteral()}\"").copy(breadCrumb = "STATUS", failureReason = FailureReason.StatusMismatch)) - else - MatchSuccess(parameters) - } + status -> MatchSuccess(parameters) else -> MatchFailure(mismatchResult("status $status", "status ${response.status}").copy(breadCrumb = "STATUS", failureReason = FailureReason.StatusMismatch)) } } diff --git a/core/src/main/kotlin/in/specmatic/core/Scenario.kt b/core/src/main/kotlin/in/specmatic/core/Scenario.kt index 0e90bac20..0c7d11536 100644 --- a/core/src/main/kotlin/in/specmatic/core/Scenario.kt +++ b/core/src/main/kotlin/in/specmatic/core/Scenario.kt @@ -8,6 +8,7 @@ import `in`.specmatic.core.utilities.exceptionCauseMessage import `in`.specmatic.core.utilities.mapZip import `in`.specmatic.core.value.* import `in`.specmatic.stub.RequestContext +import `in`.specmatic.test.ContractTest import `in`.specmatic.test.TestExecutor object ContractAndStubMismatchMessages : MismatchMessages { @@ -221,13 +222,31 @@ data class Scenario( fun generateHttpRequest(flagsBased: FlagsBased = DefaultStrategies): HttpRequest = scenarioBreadCrumb(this) { httpRequestPattern.generate(flagsBased.update(Resolver(expectedFacts, false, patterns))) } + fun matches(httpRequest: HttpRequest, httpResponse: HttpResponse, mismatchMessages: MismatchMessages = DefaultMismatchMessages, unexpectedKeyCheck: UnexpectedKeyCheck? = null): Result { + val resolver = updatedResolver(mismatchMessages, unexpectedKeyCheck).copy(context = RequestContext(httpRequest)) + + return matches(httpResponse, mismatchMessages, unexpectedKeyCheck, resolver) + } + fun matches(httpResponse: HttpResponse, mismatchMessages: MismatchMessages = DefaultMismatchMessages, unexpectedKeyCheck: UnexpectedKeyCheck? = null): Result { - val resolver = Resolver(expectedFacts, false, patterns).copy(mismatchMessages = mismatchMessages).let { - if(unexpectedKeyCheck != null) + val resolver = updatedResolver(mismatchMessages, unexpectedKeyCheck) + + return matches(httpResponse, mismatchMessages, unexpectedKeyCheck, resolver) + } + + private fun updatedResolver( + mismatchMessages: MismatchMessages, + unexpectedKeyCheck: UnexpectedKeyCheck? + ): Resolver { + return Resolver(expectedFacts, false, patterns).copy(mismatchMessages = mismatchMessages).let { + if (unexpectedKeyCheck != null) it.copy(findKeyErrorCheck = it.findKeyErrorCheck.copy(unexpectedKeyCheck = unexpectedKeyCheck)) else it } + } + + fun matches(httpResponse: HttpResponse, mismatchMessages: MismatchMessages = DefaultMismatchMessages, unexpectedKeyCheck: UnexpectedKeyCheck? = null, resolver: Resolver): Result { if (this.isNegative) { return if (is4xxResponse(httpResponse)) { @@ -572,57 +591,3 @@ object ContractAndResponseMismatch : MismatchMessages { } named $keyName in the specification was not found in the response" } } - -fun executeTest(testScenario: Scenario, testExecutor: TestExecutor, resolverStrategies: FlagsBased = DefaultStrategies): Result { - return executeTestAndReturnResultAndResponse(testScenario, testExecutor, resolverStrategies).first -} - -fun executeTestAndReturnResultAndResponse( - testScenario: Scenario, - testExecutor: TestExecutor, - flagsBased: FlagsBased -): Pair { - val request = testScenario.generateHttpRequest(flagsBased) - - return try { - testExecutor.setServerState(testScenario.serverState) - - testExecutor.preExecuteScenario(testScenario, request) - - val response = testExecutor.execute(request) - - val result = testResult(response, testScenario, flagsBased) - - Pair(result.withBindings(testScenario.bindings, response), response) - } catch (exception: Throwable) { - Pair(Result.Failure(exceptionCauseMessage(exception)) - .also { failure -> failure.updateScenario(testScenario) }, null) - } -} - -private fun testResult( - response: HttpResponse, - testScenario: Scenario, - flagsBased: FlagsBased? = null -): Result { - - val result = when { - response.specmaticResultHeaderValue() == "failure" -> Result.Failure(response.body.toStringLiteral()) - .updateScenario(testScenario) - response.body is JSONObjectValue && ignorable(response.body) -> Result.Success() - else -> testScenario.matches(response, ContractAndResponseMismatch, flagsBased?.unexpectedKeyCheck ?: ValidateUnexpectedKeys) - }.also { result -> - if (result is Result.Success && result.isPartialSuccess()) { - logger.log(" PARTIAL SUCCESS: ${result.partialSuccessMessage}") - logger.newLine() - } - } - - return result -} - -fun ignorable(body: JSONObjectValue): Boolean { - return Flags.customResponse() && - (body.findFirstChildByPath("resultStatus.status")?.toStringLiteral() == "FAILED" && - (body.findFirstChildByPath("resultStatus.errorCode")?.toStringLiteral() == "INVALID_REQUEST")) -} diff --git a/core/src/main/kotlin/in/specmatic/test/ContractTest.kt b/core/src/main/kotlin/in/specmatic/test/ContractTest.kt index 085bda351..6924018c7 100644 --- a/core/src/main/kotlin/in/specmatic/test/ContractTest.kt +++ b/core/src/main/kotlin/in/specmatic/test/ContractTest.kt @@ -2,9 +2,17 @@ package `in`.specmatic.test import `in`.specmatic.core.HttpResponse import `in`.specmatic.core.Result +import `in`.specmatic.core.Scenario + +interface ResponseValidator { + fun validate(scenario: Scenario, httpResponse: HttpResponse): Result? +} interface ContractTest { fun testResultRecord(result: Result, response: HttpResponse?): TestResultRecord? fun testDescription(): String fun runTest(testBaseURL: String, timeOut: Int): Pair + fun runTest(testExecutor: TestExecutor): Pair + + fun plusValidator(validator: ResponseValidator): ContractTest } diff --git a/core/src/main/kotlin/in/specmatic/test/ScenarioAsTest.kt b/core/src/main/kotlin/in/specmatic/test/ScenarioAsTest.kt new file mode 100644 index 000000000..6433e026a --- /dev/null +++ b/core/src/main/kotlin/in/specmatic/test/ScenarioAsTest.kt @@ -0,0 +1,124 @@ +package `in`.specmatic.test + +import `in`.specmatic.conversions.convertPathParameterStyle +import `in`.specmatic.core.* +import `in`.specmatic.core.log.HttpLogMessage +import `in`.specmatic.core.log.LogMessage +import `in`.specmatic.core.log.logger +import `in`.specmatic.core.utilities.exceptionCauseMessage + +data class ScenarioAsTest( + val scenario: Scenario, + private val flagsBased: FlagsBased, + private val sourceProvider: String? = null, + private val sourceRepository: String? = null, + private val sourceRepositoryBranch: String? = null, + private val specification: String? = null, + private val serviceType: String? = null, + private val annotations: String? = null, + private val validators: List = emptyList() +) : ContractTest { + override fun testResultRecord(result: Result, response: HttpResponse?): TestResultRecord { + val resultStatus = result.testResult() + + val responseStatus = scenario.getStatus(response) + return TestResultRecord( + convertPathParameterStyle(scenario.path), + scenario.method, + responseStatus, + resultStatus, + sourceProvider, + sourceRepository, + sourceRepositoryBranch, + specification, + serviceType + ) + } + + override fun testDescription(): String { + return scenario.testDescription() + } + + override fun runTest(testBaseURL: String, timeOut: Int): Pair { + val log: (LogMessage) -> Unit = { logMessage -> + logger.log(logMessage.withComment(this.annotations)) + } + + val httpClient = HttpClient(testBaseURL, log = log, timeout = timeOut) + + return runTest(httpClient) + } + + override fun runTest(testExecutor: TestExecutor): Pair { + + val (result, response) = executeTestAndReturnResultAndResponse(scenario, testExecutor, flagsBased) + return Pair(result.updateScenario(scenario), response) + } + + override fun plusValidator(validator: ResponseValidator): ScenarioAsTest { + return this.copy( + validators = this.validators.plus(validator) + ) + } + + private fun logComment() { + if (annotations != null) { + logger.log(annotations) + } + } + + private fun executeTestAndReturnResultAndResponse( + testScenario: Scenario, + testExecutor: TestExecutor, + flagsBased: FlagsBased + ): Pair { + val request = testScenario.generateHttpRequest(flagsBased) + + return try { + testExecutor.setServerState(testScenario.serverState) + + testExecutor.preExecuteScenario(testScenario, request) + + val response = testExecutor.execute(request) + + val validatorResult = validators.asSequence().map { it.validate(scenario, response) }.filterNotNull().firstOrNull() + val result = validatorResult ?: testResult(request, response, testScenario, flagsBased) + + Pair(result.withBindings(testScenario.bindings, response), response) + } catch (exception: Throwable) { + Pair( + Result.Failure(exceptionCauseMessage(exception)) + .also { failure -> failure.updateScenario(testScenario) }, null) + } + } + + private fun testResult( + request: HttpRequest, + response: HttpResponse, + testScenario: Scenario, + flagsBased: FlagsBased? = null + ): Result { + + val result = when { + response.specmaticResultHeaderValue() == "failure" -> Result.Failure(response.body.toStringLiteral()) + .updateScenario(testScenario) + else -> testScenario.matches(request, response, ContractAndResponseMismatch, flagsBased?.unexpectedKeyCheck ?: ValidateUnexpectedKeys) + } + + if (result is Result.Success && result.isPartialSuccess()) { + logger.log(" PARTIAL SUCCESS: ${result.partialSuccessMessage}") + logger.newLine() + } + + return result + } + +} + +private fun LogMessage.withComment(comment: String?): LogMessage { + return if (this is HttpLogMessage) { + this.copy(comment = comment) + } else { + this + } +} diff --git a/core/src/main/kotlin/in/specmatic/test/ScenarioTest.kt b/core/src/main/kotlin/in/specmatic/test/ScenarioTest.kt deleted file mode 100644 index 23315c107..000000000 --- a/core/src/main/kotlin/in/specmatic/test/ScenarioTest.kt +++ /dev/null @@ -1,64 +0,0 @@ -package `in`.specmatic.test - -import `in`.specmatic.conversions.convertPathParameterStyle -import `in`.specmatic.core.* -import `in`.specmatic.core.log.HttpLogMessage -import `in`.specmatic.core.log.LogMessage -import `in`.specmatic.core.log.logger - -class ScenarioTest( - val scenario: Scenario, - private val flagsBased: FlagsBased, - private val sourceProvider: String? = null, - private val sourceRepository: String? = null, - private val sourceRepositoryBranch: String? = null, - private val specification: String? = null, - private val serviceType: String? = null, - private val annotations: String? = null -) : ContractTest { - override fun testResultRecord(result: Result, response: HttpResponse?): TestResultRecord { - val resultStatus = result.testResult() - - val responseStatus = scenario.getStatus(response) - return TestResultRecord( - convertPathParameterStyle(scenario.path), - scenario.method, - responseStatus, - resultStatus, - sourceProvider, - sourceRepository, - sourceRepositoryBranch, - specification, - serviceType - ) - } - - override fun testDescription(): String { - return scenario.testDescription() - } - - override fun runTest(testBaseURL: String, timeOut: Int): Pair { - val log: (LogMessage) -> Unit = { logMessage -> - logger.log(logMessage.withComment(this.annotations)) - } - - val httpClient = HttpClient(testBaseURL, log = log, timeout = timeOut) - val (result, response) = executeTestAndReturnResultAndResponse(scenario, httpClient, flagsBased) - return Pair(result.updateScenario(scenario), response) - } - - private fun logComment() { - if (annotations != null) { - logger.log(annotations) - } - } - -} - -private fun LogMessage.withComment(comment: String?): LogMessage { - return if (this is HttpLogMessage) { - this.copy(comment = comment) - } else { - this - } -} diff --git a/core/src/main/kotlin/in/specmatic/test/ScenarioTestGenerationException.kt b/core/src/main/kotlin/in/specmatic/test/ScenarioTestGenerationException.kt index cfec30710..b6c91ca0f 100644 --- a/core/src/main/kotlin/in/specmatic/test/ScenarioTestGenerationException.kt +++ b/core/src/main/kotlin/in/specmatic/test/ScenarioTestGenerationException.kt @@ -17,6 +17,18 @@ class ScenarioTestGenerationException(val scenario: Scenario, val e: Throwable, } override fun runTest(testBaseURL: String, timeOut: Int): Pair { + return error() + } + + override fun runTest(testExecutor: TestExecutor): Pair { + return error() + } + + override fun plusValidator(validator: ResponseValidator): ContractTest { + return this + } + + fun error(): Pair { val result: Result = when(e) { is ContractException -> Result.Failure(message, e.failure(), breadCrumb = breadCrumb ?: "").updateScenario(scenario) else -> Result.Failure(message + " - " + exceptionCauseMessage(e), breadCrumb = breadCrumb ?: "").updateScenario(scenario) diff --git a/core/src/main/kotlin/in/specmatic/test/ScenarioTestGenerationFailure.kt b/core/src/main/kotlin/in/specmatic/test/ScenarioTestGenerationFailure.kt index 6e73483a4..3434fdb8a 100644 --- a/core/src/main/kotlin/in/specmatic/test/ScenarioTestGenerationFailure.kt +++ b/core/src/main/kotlin/in/specmatic/test/ScenarioTestGenerationFailure.kt @@ -18,4 +18,12 @@ class ScenarioTestGenerationFailure(val scenario: Scenario, val failure: Result. return Pair(failure.updateScenario(scenario), null) } + override fun runTest(testExecutor: TestExecutor): Pair { + return Pair(failure.updateScenario(scenario), null) + } + + override fun plusValidator(validator: ResponseValidator): ContractTest { + return this + } + } \ No newline at end of file diff --git a/core/src/test/kotlin/in/specmatic/conversions/OpenApiIntegrationTest.kt b/core/src/test/kotlin/in/specmatic/conversions/OpenApiIntegrationTest.kt index 6297bb5a2..8fdb48726 100644 --- a/core/src/test/kotlin/in/specmatic/conversions/OpenApiIntegrationTest.kt +++ b/core/src/test/kotlin/in/specmatic/conversions/OpenApiIntegrationTest.kt @@ -46,8 +46,8 @@ Examples: """.trimIndent(), sourceSpecPath ) - val contractTests = contract.generateContractTestScenarios(emptyList()).map { it.second.value } - val result = executeTest(contractTests.single(), object : TestExecutor { + val contractTests = contract.generateContractTests(emptyList()) + val result = contractTests.single().runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.headers).containsEntry(HttpHeaders.AUTHORIZATION, "Bearer abc123") return HttpResponse.ok("success") @@ -57,7 +57,7 @@ Examples: } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -66,8 +66,8 @@ Examples: fun `should generate test with oauth2 authorization code security scheme with random token in authorization header when no example exists`() { val feature = parseContractFileToFeature("./src/test/resources/openapi/hello_with_oauth2_authorization_code_flow.yaml") - val contractTests = feature.generateContractTestScenarios(emptyList()).map { it.second.value } - val result = executeTest(contractTests.single(), object : TestExecutor { + val contractTests = feature.generateContractTests(emptyList()) + val result = contractTests.single().runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.headers).containsKey(HttpHeaders.AUTHORIZATION) assertThat(request.headers[HttpHeaders.AUTHORIZATION]).matches("Bearer (\\S+)") @@ -78,7 +78,7 @@ Examples: } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -89,8 +89,8 @@ Examples: "./src/test/resources/openapi/hello_with_oauth2_authorization_code_flow.yaml", securityConfiguration = newSecurityConfiguration(token) ) - val contractTests = feature.generateContractTestScenarios(emptyList()).map { it.second.value } - val result = executeTest(contractTests.single(), object : TestExecutor { + val contractTests = feature.generateContractTests(emptyList()) + val result = contractTests.single().runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.headers).containsKey(HttpHeaders.AUTHORIZATION) assertThat(request.headers[HttpHeaders.AUTHORIZATION]).matches("Bearer $token") @@ -100,7 +100,7 @@ Examples: override fun setServerState(serverState: Map) { } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -114,8 +114,8 @@ Examples: "./src/test/resources/openapi/hello_with_oauth2_authorization_code_flow.yaml", environmentAndPropertiesConfiguration = environmentAndPropertiesConfiguration ) - val contractTests = feature.generateContractTestScenarios(emptyList()).map { it.second.value } - val result = executeTest(contractTests.single(), object : TestExecutor { + val contractTests = feature.generateContractTests(emptyList()) + val result = contractTests.single().runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.headers).containsKey(HttpHeaders.AUTHORIZATION) assertThat(request.headers[HttpHeaders.AUTHORIZATION]).matches("Bearer ENV1234") @@ -125,7 +125,7 @@ Examples: override fun setServerState(serverState: Map) { } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -219,8 +219,8 @@ Feature: Authenticated | Bearer abc123 | 10 | """.trimIndent(), sourceSpecPath ) - val contractTests = contract.generateContractTestScenarios(emptyList()).map { it.second.value } - val result = executeTest(contractTests.single(), object : TestExecutor { + val contractTests = contract.generateContractTests(emptyList()) + val result = contractTests.single().runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.headers).containsEntry(HttpHeaders.AUTHORIZATION, "Bearer abc123") return HttpResponse.ok("success") @@ -230,7 +230,7 @@ Feature: Authenticated } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -271,8 +271,8 @@ Feature: Authenticated | Bearer abc123 | 10 | """.trimIndent(), sourceSpecPath ) - val contractTests = contract.generateContractTestScenarios(emptyList()).map { it.second.value } - val result = executeTest(contractTests.single(), object : TestExecutor { + val contractTests = contract.generateContractTests(emptyList()) + val result = contractTests.single().runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.headers).containsEntry(HttpHeaders.AUTHORIZATION, "Bearer abc123") return HttpResponse.ok("success") @@ -282,7 +282,7 @@ Feature: Authenticated } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -323,8 +323,8 @@ Feature: Authenticated | Bearer abc123 | 10 | """.trimIndent(), sourceSpecPath ) - val contractTests = contract.generateContractTestScenarios(emptyList()).map { it.second.value } - val result = executeTest(contractTests.single(), object : TestExecutor { + val contractTests = contract.generateContractTests(emptyList()) + val result = contractTests.single().runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.headers).containsEntry(HttpHeaders.AUTHORIZATION, "Bearer abc123") return HttpResponse.ok("success") @@ -334,7 +334,7 @@ Feature: Authenticated } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -376,8 +376,8 @@ Feature: Authenticated """.trimIndent(), sourceSpecPath ) - val contractTests = contract.generateContractTestScenarios(emptyList()).map { it.second.value } - val result = executeTest(contractTests.single(), object : TestExecutor { + val contractTests = contract.generateContractTests(emptyList()) + val result = contractTests.single().runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.headers).containsEntry("Authorization", "Bearer abc123") return HttpResponse.ok("success") @@ -387,7 +387,7 @@ Feature: Authenticated } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -395,10 +395,10 @@ Feature: Authenticated @Test fun `should generate test with bearer security scheme with random token in authorization header when no example exists`() { val feature = parseContractFileToFeature("./src/test/resources/openapi/authenticated.yaml") - val contractTests = feature.generateContractTestScenarios(emptyList()).map { it.second.value } + val contractTests = feature.generateContractTests(emptyList()) var requestMadeWithRandomlyGeneratedBearerToken = false contractTests.forEach { scenario -> - val result = executeTest(scenario, object : TestExecutor { + val result = scenario.runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { request.headers[HttpHeaders.AUTHORIZATION]?.takeIf { it.matches(Regex("Bearer (\\S+)")) @@ -411,7 +411,7 @@ Feature: Authenticated override fun setServerState(serverState: Map) { } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } assertThat(requestMadeWithRandomlyGeneratedBearerToken).isTrue @@ -424,10 +424,10 @@ Feature: Authenticated "./src/test/resources/openapi/authenticated.yaml", securityConfiguration = securityConfigurationForBearerScheme(token) ) - val contractTests = feature.generateContractTestScenarios(emptyList()).map { it.second.value } + val contractTests = feature.generateContractTests(emptyList()) var requestMadeWithTokenFromSpecmaticJson = false contractTests.forEach { scenario -> - val result = executeTest(scenario, object : TestExecutor { + val result = scenario.runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { request.headers[HttpHeaders.AUTHORIZATION]?.takeIf { it == "Bearer $token" @@ -440,7 +440,7 @@ Feature: Authenticated override fun setServerState(serverState: Map) { } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } assertThat(requestMadeWithTokenFromSpecmaticJson).isTrue @@ -460,10 +460,10 @@ Feature: Authenticated "./src/test/resources/openapi/authenticated.yaml", environmentAndPropertiesConfiguration = environmentAndPropertiesConfiguration ) - val contractTests = feature.generateContractTestScenarios(emptyList()).map { it.second.value } + val contractTests = feature.generateContractTests(emptyList()) var requestMadeWithTokenFromSpecmaticJson = false contractTests.forEach { scenario -> - val result = executeTest(scenario, object : TestExecutor { + val result = scenario.runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { request.headers[HttpHeaders.AUTHORIZATION]?.takeIf { it == "Bearer ENV1234" @@ -476,7 +476,7 @@ Feature: Authenticated override fun setServerState(serverState: Map) { } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } assertThat(requestMadeWithTokenFromSpecmaticJson).isTrue @@ -711,8 +711,8 @@ Feature: Authenticated """.trimIndent(), sourceSpecPath ) - val contractTests = contract.generateContractTestScenarios(emptyList()).map { it.second.value } - val result = executeTest(contractTests.single(), object : TestExecutor { + val contractTests = contract.generateContractTests(emptyList()) + val result = contractTests.single().runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.queryParams.containsEntry("apiKey", "abc123")).isTrue return HttpResponse.ok("success") @@ -722,7 +722,7 @@ Feature: Authenticated } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -774,8 +774,8 @@ Feature: Authenticated """.trimIndent(), sourceSpecPath ) - val contractTests = contract.generateContractTestScenarios(emptyList()).map { it.second.value } - val result = executeTest(contractTests.single(), object : TestExecutor { + val contractTests = contract.generateContractTests(emptyList()) + val result = contractTests.single().runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.headers).containsEntry("X-API-KEY", "abc123") return HttpResponse.ok("success") @@ -785,7 +785,7 @@ Feature: Authenticated } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -797,10 +797,10 @@ Feature: Authenticated "./src/test/resources/openapi/authenticated.yaml", securityConfiguration = securityConfigurationForApiKeyInHeaderScheme(token) ) - val contractTests = feature.generateContractTestScenarios(emptyList()).map { it.second.value } + val contractTests = feature.generateContractTests(emptyList()) var requestMadeWithApiKeyInHeaderFromSpecmaticJson = false contractTests.forEach { scenario -> - val result = executeTest(scenario, object : TestExecutor { + val result = scenario.runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { request.headers["X-API-KEY"]?.takeIf { it == token @@ -813,7 +813,7 @@ Feature: Authenticated override fun setServerState(serverState: Map) { } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } assertThat(requestMadeWithApiKeyInHeaderFromSpecmaticJson).isTrue @@ -834,10 +834,10 @@ Feature: Authenticated "./src/test/resources/openapi/authenticated.yaml", environmentAndPropertiesConfiguration = environmentAndPropertiesConfiguration ) - val contractTests = feature.generateContractTestScenarios(emptyList()).map { it.second.value } + val contractTests = feature.generateContractTests(emptyList()) var requestMadeWithApiKeyInHeaderFromSpecmaticJson = false contractTests.forEach { scenario -> - val result = executeTest(scenario, object : TestExecutor { + val result = scenario.runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { request.headers["X-API-KEY"]?.takeIf { it == "ENV1234" @@ -850,7 +850,7 @@ Feature: Authenticated override fun setServerState(serverState: Map) { } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } assertThat(requestMadeWithApiKeyInHeaderFromSpecmaticJson).isTrue @@ -863,10 +863,10 @@ Feature: Authenticated "./src/test/resources/openapi/authenticated.yaml", securityConfiguration = securityConfigurationForApiKeyInQueryScheme(token) ) - val contractTests = feature.generateContractTestScenarios(emptyList()).map { it.second.value } + val contractTests = feature.generateContractTests(emptyList()) var requestMadeWithApiKeyInQueryFromSpecmaticJson = false contractTests.forEach { scenario -> - val result = executeTest(scenario, object : TestExecutor { + val result = scenario.runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { if (request.queryParams.containsKey("apiKey")) { request.queryParams.getValues("apiKey").first().takeIf { @@ -881,7 +881,7 @@ Feature: Authenticated override fun setServerState(serverState: Map) { } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } assertThat(requestMadeWithApiKeyInQueryFromSpecmaticJson).isTrue diff --git a/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt b/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt index 346877d4b..1d79fb3af 100644 --- a/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt +++ b/core/src/test/kotlin/in/specmatic/conversions/OpenApiKtTest.kt @@ -747,8 +747,8 @@ Feature: multipart file upload """.trimIndent(), sourceSpecPath ) - val contractTests = contract.generateContractTestScenarios(emptyList()).map { it.second.value } - val result = executeTest(contractTests.single(), object : TestExecutor { + val contractTests = contract.generateContractTests(emptyList()) + val result = contractTests.single().runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { val multipartFileValues = request.multiPartFormData.filterIsInstance() assertThat(multipartFileValues.size).isEqualTo(1) @@ -761,7 +761,7 @@ Feature: multipart file upload } - }) + }).first assertThat(result).isInstanceOf(Result.Success::class.java) } @@ -1385,7 +1385,7 @@ Background: override fun setServerState(serverState: Map) { } }, - scenarioNames = listOf("create a pet. Response: pet response") + testDescriptionFilter = listOf("POST /pets -> 201") ) assertFalse(results.success()) @@ -1454,7 +1454,7 @@ Background: override fun setServerState(serverState: Map) { } }, - scenarioNames = listOf("create a pet. Response: pet response") + testDescriptionFilter = listOf("POST /pets -> 201") ) assertFalse(results.success()) @@ -1518,7 +1518,7 @@ Background: override fun setServerState(serverState: Map) { } }, - scenarioNames = listOf("create a pet. Response: pet response") + testDescriptionFilter = listOf("POST /pets -> 201") ) assertFalse(results.success()) @@ -1670,17 +1670,18 @@ Scenario: zero should return not found var executed = false - val result = executeTest(feature.scenarios.first(), object : TestExecutor { - override fun execute(request: HttpRequest): HttpResponse { - executed = true - return if (request.queryParams.keys.containsAll(listOf("name", "message"))) HttpResponse.OK - else HttpResponse.ERROR_400 - } + val result = `in`.specmatic.test.ScenarioAsTest(feature.scenarios.first(), DefaultStrategies) + .runTest(object : TestExecutor { + override fun execute(request: HttpRequest): HttpResponse { + executed = true + return if (request.queryParams.keys.containsAll(listOf("name", "message"))) HttpResponse.OK + else HttpResponse.ERROR_400 + } - override fun setServerState(serverState: Map) { + override fun setServerState(serverState: Map) { - } - }) + } + }).first assertThat(result).isInstanceOf(Result.Success::class.java) assertThat(executed).isTrue diff --git a/core/src/test/kotlin/in/specmatic/conversions/OpenApiSpecificationTest.kt b/core/src/test/kotlin/in/specmatic/conversions/OpenApiSpecificationTest.kt index 98054dcac..a1141e681 100644 --- a/core/src/test/kotlin/in/specmatic/conversions/OpenApiSpecificationTest.kt +++ b/core/src/test/kotlin/in/specmatic/conversions/OpenApiSpecificationTest.kt @@ -5437,8 +5437,8 @@ paths: val feature = OpenApiSpecification.fromYAML(contractString, "").toFeature() val results: List = - feature.generateContractTestScenarios(emptyList()).toList().map { it.second.value }.map { - executeTest(it, object : TestExecutor { + feature.generateContractTests(emptyList()).toList().map { + it.runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.body).isInstanceOf(JSONObjectValue::class.java) @@ -5450,7 +5450,7 @@ paths: override fun setServerState(serverState: Map) { } - }) + }).first } assertThat(results).hasSize(1) @@ -5504,8 +5504,8 @@ paths: val feature = OpenApiSpecification.fromYAML(contractString, "").toFeature() val results: List = - feature.generateContractTestScenarios(emptyList()).toList().map { it.second.value }.map { - executeTest(it, object : TestExecutor { + feature.generateContractTests(emptyList()).toList().map { + it.runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.formFields).containsKey("Data") @@ -5524,7 +5524,7 @@ paths: override fun setServerState(serverState: Map) { } - }) + }).first } assertThat(results).hasSize(1) @@ -5574,8 +5574,8 @@ paths: val feature = OpenApiSpecification.fromYAML(contractString, "").toFeature() val results: List = - feature.generateContractTestScenarios(emptyList()).toList().map { it.second.value }.map { - executeTest(it, object : TestExecutor { + feature.generateContractTests(emptyList()).toList().map { + it.runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.formFields).containsKey("Data") assertThat(request.formFields["Data"]).isEqualTo("abc123") @@ -5586,7 +5586,7 @@ paths: override fun setServerState(serverState: Map) { } - }) + }).first } assertThat(results).hasSize(1) @@ -5634,8 +5634,8 @@ paths: val feature = OpenApiSpecification.fromYAML(contractString, "").toFeature() val results: List = - feature.generateContractTestScenarios(emptyList()).toList().map { it.second.value }.map { - executeTest(it, object : TestExecutor { + feature.generateContractTests(emptyList()).toList().map { + it.runTest(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { assertThat(request.multiPartFormData.first().name).isEqualTo("Data") @@ -5648,7 +5648,7 @@ paths: override fun setServerState(serverState: Map) { } - }) + }).first } assertThat(results).hasSize(1) diff --git a/core/src/test/kotlin/in/specmatic/conversions/RegexSupportTest.kt b/core/src/test/kotlin/in/specmatic/conversions/RegexSupportTest.kt index 8aa4d715e..f4c2f45ec 100644 --- a/core/src/test/kotlin/in/specmatic/conversions/RegexSupportTest.kt +++ b/core/src/test/kotlin/in/specmatic/conversions/RegexSupportTest.kt @@ -212,12 +212,10 @@ class RegexSupportTest { } } - assertThatThrownBy { feature.executeTests(mockTestClient) }.satisfies( - Consumer { - assertThat(it).isInstanceOf(ContractException::class.java) - assertThat(exceptionCauseMessage(it)).contains(regex) - } - ) + val results = feature.executeTests(mockTestClient) + + assertThat(results.success()).isFalse() + assertThat(results.report()).contains(regex) } @Nested diff --git a/core/src/test/kotlin/in/specmatic/core/ContractAsTestWithSamplesInTable.kt b/core/src/test/kotlin/in/specmatic/core/ContractAsTestWithSamplesInTable.kt index b2b8a37e8..b1e3f0773 100644 --- a/core/src/test/kotlin/in/specmatic/core/ContractAsTestWithSamplesInTable.kt +++ b/core/src/test/kotlin/in/specmatic/core/ContractAsTestWithSamplesInTable.kt @@ -1,12 +1,10 @@ package `in`.specmatic.core -import `in`.specmatic.core.pattern.ContractException import `in`.specmatic.core.pattern.NumberPattern import `in`.specmatic.core.pattern.StringPattern import `in`.specmatic.core.value.* import `in`.specmatic.test.TestExecutor import org.assertj.core.api.Assertions.assertThat -import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -28,20 +26,16 @@ class ContractAsTestWithSamplesInTable { | 10 | 20 | 30 | | hello | 30 | 40 | """ - Assertions.assertThrows(ContractException::class.java) { jsonResponsesTestsShouldBeVerifiedAgainstTable(contractGherkin) } - } - @Throws(Throwable::class) - private fun jsonResponsesTestsShouldBeVerifiedAgainstTable(contractGherkin: String) { val contractBehaviour = parseGherkinStringToFeature(contractGherkin) val results = contractBehaviour.executeTests(object : TestExecutor { override fun execute(request: HttpRequest): HttpResponse { val accountId = request.queryParams.getOrElse("account_id") { - val pathParts = request.path!!.split("/".toRegex()).toTypedArray() + val pathParts = request.path!!.split("/".toRegex()).toTypedArray() pathParts[pathParts.size - 1] } assertEquals("GET", request.method) - assertTrue( NumberPattern().matches(NumberValue(accountId.toInt()), Resolver()) is Result.Success) + assertTrue(NumberPattern().matches(NumberValue(accountId.toInt()), Resolver()) is Result.Success) val headers: HashMap = object : HashMap() { init { put("Content-Type", "application/json") @@ -59,7 +53,9 @@ class ContractAsTestWithSamplesInTable { override fun setServerState(serverState: Map) {} }) - assertThat(results.success()).isTrue() + assertThat(results.successCount).isOne() + assertThat(results.failureCount).isOne() + assertThat(results.success()).withFailMessage(results.report()).isFalse() } @Test @@ -249,7 +245,7 @@ Feature: Contract for /balance API } } - val xmlResponseString: String = when(name.toStringLiteral()) { + val xmlResponseString: String = when (name.toStringLiteral()) { "John Doe" -> "10" "Jane Doe" -> "20" else -> fail("Expected name to be either \"John Doe\" or \"Jane Doe\", got ${name.toStringLiteral()}") diff --git a/core/src/test/kotlin/in/specmatic/core/ScenarioTest.kt b/core/src/test/kotlin/in/specmatic/core/ScenarioAsTest.kt similarity index 97% rename from core/src/test/kotlin/in/specmatic/core/ScenarioTest.kt rename to core/src/test/kotlin/in/specmatic/core/ScenarioAsTest.kt index d1bcc99bb..03323e398 100644 --- a/core/src/test/kotlin/in/specmatic/core/ScenarioTest.kt +++ b/core/src/test/kotlin/in/specmatic/core/ScenarioAsTest.kt @@ -14,7 +14,7 @@ import org.junit.jupiter.api.Test import java.util.* import java.util.function.Consumer -internal class ScenarioTest { +internal class ScenarioAsTest { @Test fun `should generate one test scenario when there are no examples`() { val scenario = Scenario( @@ -513,16 +513,17 @@ paths: """.trimIndent(), "" ).toFeature() - val contractTestScenarios = contract.generateContractTestScenarios(emptyList()).map { it.second.value } + val contractTestScenarios = contract.generateContractTests(emptyList()) - val result: Result = executeTest(contractTestScenarios.first(), object : TestExecutor { - override fun execute(request: HttpRequest): HttpResponse { - return HttpResponse.ok("abc") - } + val result: Result = + contractTestScenarios.first().runTest(object : TestExecutor { + override fun execute(request: HttpRequest): HttpResponse { + return HttpResponse.ok("abc") + } - override fun setServerState(serverState: Map) { - } - }) as Result.Failure + override fun setServerState(serverState: Map) { + } + }).first as Result.Failure assertThat(result.reportString()).contains("Contract expected") assertThat(result.reportString()).contains("response contained") diff --git a/core/src/test/kotlin/integration_tests/LoadTestsFromExternalisedFiles.kt b/core/src/test/kotlin/integration_tests/LoadTestsFromExternalisedFiles.kt index f38c38536..6870a5af5 100644 --- a/core/src/test/kotlin/integration_tests/LoadTestsFromExternalisedFiles.kt +++ b/core/src/test/kotlin/integration_tests/LoadTestsFromExternalisedFiles.kt @@ -74,30 +74,26 @@ class LoadTestsFromExternalisedFiles { fun `externalized tests should be validated`() { val feature = OpenApiSpecification.fromFile("src/test/resources/openapi/has_invalid_externalized_test.yaml").toFeature().loadExternalisedExamples() - assertThatThrownBy { - feature.executeTests(object : TestExecutor { - override fun execute(request: HttpRequest): HttpResponse { - assertThat(request.path).isEqualTo("/order_action_figure") - assertThat(request.method).isEqualTo("POST") - assertThat(request.body).isEqualTo(parsedJSONObject("""{"name": "Master Yoda", "description": "Head of the Jedi Council"}""")) + val results = feature.executeTests(object : TestExecutor { + override fun execute(request: HttpRequest): HttpResponse { + assertThat(request.path).isEqualTo("/order_action_figure") + assertThat(request.method).isEqualTo("POST") + assertThat(request.body).isEqualTo(parsedJSONObject("""{"name": "Master Yoda", "description": "Head of the Jedi Council"}""")) - return HttpResponse.ok(parsedJSONObject("""{"id": 1}""")) - } + return HttpResponse.ok(parsedJSONObject("""{"id": 1}""")) + } - override fun setServerState(serverState: Map) { - } - }) - }.satisfies(Consumer { - assertThat(it).isInstanceOf(ContractException::class.java) - it as ContractException + override fun setServerState(serverState: Map) { + } + }) - assertThat(it.report()) - .contains(">> REQUEST.BODY.description") - .contains("10") + println(results.report()) - println(it.report()) - }) + assertThat(results.report()) + .contains(">> REQUEST.BODY.description") + .contains("10") + assertThat(results.success()).isFalse() } @Test diff --git a/version.properties b/version.properties index dc9939ae1..36a3f2764 100644 --- a/version.properties +++ b/version.properties @@ -1 +1 @@ -version=1.3.32 +version=1.3.33