Skip to content

Commit

Permalink
Merge pull request #1161 from znsio/changes_for_plugins
Browse files Browse the repository at this point in the history
  • Loading branch information
joelrosario authored Jul 1, 2024
2 parents 906d33e + f228a0c commit ca7e622
Show file tree
Hide file tree
Showing 18 changed files with 309 additions and 266 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ data class EnvironmentAndPropertiesConfiguration(val environmentVariables: Map<S
constructor() : this(System.getenv(), System.getProperties().toMap())

val VALIDATE_RESPONSE_VALUE = "VALIDATE_RESPONSE_VALUE"
private val CUSTOM_RESPONSE_NAME = "CUSTOM_RESPONSE"
val SPECMATIC_GENERATIVE_TESTS = "SPECMATIC_GENERATIVE_TESTS"
private val MAX_TEST_REQUEST_COMBINATIONS = "MAX_TEST_REQUEST_COMBINATIONS"
val SCHEMA_EXAMPLE_DEFAULT = "SCHEMA_EXAMPLE_DEFAULT"
Expand All @@ -30,10 +29,6 @@ data class EnvironmentAndPropertiesConfiguration(val environmentVariables: Map<S
return environmentVariables[flagName] ?: systemProperties[flagName]?.toString() ?: System.getenv(flagName) ?: System.getProperty(flagName)
}

fun customResponse(): Boolean {
return flagValue(CUSTOM_RESPONSE_NAME) == "true"
}

private fun booleanFlag(flagName: String, default: String = "false") = BooleanUtils.toBoolean(flagValue(flagName) ?: default)

fun extensibleSchema(): Boolean {
Expand Down
8 changes: 3 additions & 5 deletions core/src/main/kotlin/in/specmatic/core/Contract.kt
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
package `in`.specmatic.core

import `in`.specmatic.core.pattern.ContractException
import `in`.specmatic.stub.HttpStub
import `in`.specmatic.test.HttpClient

data class Contract(val contract: Feature) {
data class Contract(val feature: Feature) {
companion object {
fun fromGherkin(contractGherkin: String): Contract {
return Contract(parseGherkinStringToFeature(contractGherkin))
Expand All @@ -14,11 +13,10 @@ data class Contract(val contract: Feature) {
fun samples(fake: HttpStub) = samples(fake.endPoint)

private fun samples(endPoint: String) {
val contractBehaviour = contract
val httpClient = HttpClient(endPoint)

contractBehaviour.generateContractTestScenarios(emptyList()).map { it.second.value }.fold(Results()) { results, scenario ->
Results(results = results.results.plus(executeTest(scenario, httpClient)).toMutableList())
feature.generateContractTests(emptyList()).fold(Results()) { results, contractTest ->
Results(results = results.results.plus(contractTest.runTest(httpClient).first))
}
}
}
30 changes: 20 additions & 10 deletions core/src/main/kotlin/in/specmatic/core/Feature.kt
Original file line number Diff line number Diff line change
Expand Up @@ -189,16 +189,22 @@ data class Feature(
}

fun executeTests(
testExecutorFn: TestExecutor,
testExecutor: TestExecutor,
suggestions: List<Scenario> = emptyList(),
scenarioNames: List<String> = emptyList()
): Results =
generateContractTestScenarios(suggestions)
.map { it.second.value }
.filter { scenarioNames.isEmpty() || scenarioNames.contains(it.name) }
.fold(Results()) { results, scenario ->
Results(results = results.results.plus(executeTest(scenario, testExecutorFn, flagsBased)))
testDescriptionFilter: List<String> = emptyList()
): Results {
return generateContractTests(suggestions)
.filter { contractTest ->
testDescriptionFilter.isEmpty() ||
testDescriptionFilter.any { scenarioName ->
contractTest.testDescription().contains(scenarioName)
}
}
.fold(Results()) { results, contractTest ->
val (result, _) = contractTest.runTest(testExecutor)
Results(results = results.results.plus(result))
}
}

fun setServerState(serverState: Map<String, Value>) {
this.serverState = this.serverState.plus(serverState)
Expand Down Expand Up @@ -300,7 +306,7 @@ data class Feature(
return generateContractTestScenarios(suggestions).map { (originalScenario, returnValue) ->
returnValue.realise(
hasValue = { concreteTestScenario, comment ->
ScenarioTest(
ScenarioAsTest(
concreteTestScenario,
flagsBased,
concreteTestScenario.sourceProvider,
Expand Down Expand Up @@ -367,7 +373,11 @@ data class Feature(
negativeScenarioResult.ifHasValue { result: HasValue<Scenario> ->
val description = result.valueDetails.singleLineDescription()

HasValue(result.value.copy(descriptionFromPlugin = "${result.value.apiDescription} [${description}]"))
val tag = if(description.isNotBlank())
" [${description}]"
else
""
HasValue(result.value.copy(descriptionFromPlugin = "${result.value.apiDescription}$tag"))
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,12 +93,7 @@ data class HttpResponsePattern(
val body = response.body

return when (response.status) {
status -> {
if(Flags.customResponse() && response.status.toString().startsWith("2") && body is JSONObjectValue && body.findFirstChildByPath("resultStatus.status")?.toStringLiteral() == "FAILED")
MatchFailure(mismatchResult("status $status and resultStatus.status == \"SUCCESS\"", "status ${response.status} and resultStatus.status == \"${body.findFirstChildByPath("resultStatus.status")?.toStringLiteral()}\"").copy(breadCrumb = "STATUS", failureReason = FailureReason.StatusMismatch))
else
MatchSuccess(parameters)
}
status -> MatchSuccess(parameters)
else -> MatchFailure(mismatchResult("status $status", "status ${response.status}").copy(breadCrumb = "STATUS", failureReason = FailureReason.StatusMismatch))
}
}
Expand Down
77 changes: 21 additions & 56 deletions core/src/main/kotlin/in/specmatic/core/Scenario.kt
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import `in`.specmatic.core.utilities.exceptionCauseMessage
import `in`.specmatic.core.utilities.mapZip
import `in`.specmatic.core.value.*
import `in`.specmatic.stub.RequestContext
import `in`.specmatic.test.ContractTest
import `in`.specmatic.test.TestExecutor

object ContractAndStubMismatchMessages : MismatchMessages {
Expand Down Expand Up @@ -221,13 +222,31 @@ data class Scenario(
fun generateHttpRequest(flagsBased: FlagsBased = DefaultStrategies): HttpRequest =
scenarioBreadCrumb(this) { httpRequestPattern.generate(flagsBased.update(Resolver(expectedFacts, false, patterns))) }

fun matches(httpRequest: HttpRequest, httpResponse: HttpResponse, mismatchMessages: MismatchMessages = DefaultMismatchMessages, unexpectedKeyCheck: UnexpectedKeyCheck? = null): Result {
val resolver = updatedResolver(mismatchMessages, unexpectedKeyCheck).copy(context = RequestContext(httpRequest))

return matches(httpResponse, mismatchMessages, unexpectedKeyCheck, resolver)
}

fun matches(httpResponse: HttpResponse, mismatchMessages: MismatchMessages = DefaultMismatchMessages, unexpectedKeyCheck: UnexpectedKeyCheck? = null): Result {
val resolver = Resolver(expectedFacts, false, patterns).copy(mismatchMessages = mismatchMessages).let {
if(unexpectedKeyCheck != null)
val resolver = updatedResolver(mismatchMessages, unexpectedKeyCheck)

return matches(httpResponse, mismatchMessages, unexpectedKeyCheck, resolver)
}

private fun updatedResolver(
mismatchMessages: MismatchMessages,
unexpectedKeyCheck: UnexpectedKeyCheck?
): Resolver {
return Resolver(expectedFacts, false, patterns).copy(mismatchMessages = mismatchMessages).let {
if (unexpectedKeyCheck != null)
it.copy(findKeyErrorCheck = it.findKeyErrorCheck.copy(unexpectedKeyCheck = unexpectedKeyCheck))
else
it
}
}

fun matches(httpResponse: HttpResponse, mismatchMessages: MismatchMessages = DefaultMismatchMessages, unexpectedKeyCheck: UnexpectedKeyCheck? = null, resolver: Resolver): Result {

if (this.isNegative) {
return if (is4xxResponse(httpResponse)) {
Expand Down Expand Up @@ -572,57 +591,3 @@ object ContractAndResponseMismatch : MismatchMessages {
} named $keyName in the specification was not found in the response"
}
}

fun executeTest(testScenario: Scenario, testExecutor: TestExecutor, resolverStrategies: FlagsBased = DefaultStrategies): Result {
return executeTestAndReturnResultAndResponse(testScenario, testExecutor, resolverStrategies).first
}

fun executeTestAndReturnResultAndResponse(
testScenario: Scenario,
testExecutor: TestExecutor,
flagsBased: FlagsBased
): Pair<Result, HttpResponse?> {
val request = testScenario.generateHttpRequest(flagsBased)

return try {
testExecutor.setServerState(testScenario.serverState)

testExecutor.preExecuteScenario(testScenario, request)

val response = testExecutor.execute(request)

val result = testResult(response, testScenario, flagsBased)

Pair(result.withBindings(testScenario.bindings, response), response)
} catch (exception: Throwable) {
Pair(Result.Failure(exceptionCauseMessage(exception))
.also { failure -> failure.updateScenario(testScenario) }, null)
}
}

private fun testResult(
response: HttpResponse,
testScenario: Scenario,
flagsBased: FlagsBased? = null
): Result {

val result = when {
response.specmaticResultHeaderValue() == "failure" -> Result.Failure(response.body.toStringLiteral())
.updateScenario(testScenario)
response.body is JSONObjectValue && ignorable(response.body) -> Result.Success()
else -> testScenario.matches(response, ContractAndResponseMismatch, flagsBased?.unexpectedKeyCheck ?: ValidateUnexpectedKeys)
}.also { result ->
if (result is Result.Success && result.isPartialSuccess()) {
logger.log(" PARTIAL SUCCESS: ${result.partialSuccessMessage}")
logger.newLine()
}
}

return result
}

fun ignorable(body: JSONObjectValue): Boolean {
return Flags.customResponse() &&
(body.findFirstChildByPath("resultStatus.status")?.toStringLiteral() == "FAILED" &&
(body.findFirstChildByPath("resultStatus.errorCode")?.toStringLiteral() == "INVALID_REQUEST"))
}
8 changes: 8 additions & 0 deletions core/src/main/kotlin/in/specmatic/test/ContractTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,17 @@ package `in`.specmatic.test

import `in`.specmatic.core.HttpResponse
import `in`.specmatic.core.Result
import `in`.specmatic.core.Scenario

interface ResponseValidator {
fun validate(scenario: Scenario, httpResponse: HttpResponse): Result?
}

interface ContractTest {
fun testResultRecord(result: Result, response: HttpResponse?): TestResultRecord?
fun testDescription(): String
fun runTest(testBaseURL: String, timeOut: Int): Pair<Result, HttpResponse?>
fun runTest(testExecutor: TestExecutor): Pair<Result, HttpResponse?>

fun plusValidator(validator: ResponseValidator): ContractTest
}
124 changes: 124 additions & 0 deletions core/src/main/kotlin/in/specmatic/test/ScenarioAsTest.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
package `in`.specmatic.test

import `in`.specmatic.conversions.convertPathParameterStyle
import `in`.specmatic.core.*
import `in`.specmatic.core.log.HttpLogMessage
import `in`.specmatic.core.log.LogMessage
import `in`.specmatic.core.log.logger
import `in`.specmatic.core.utilities.exceptionCauseMessage

data class ScenarioAsTest(
val scenario: Scenario,
private val flagsBased: FlagsBased,
private val sourceProvider: String? = null,
private val sourceRepository: String? = null,
private val sourceRepositoryBranch: String? = null,
private val specification: String? = null,
private val serviceType: String? = null,
private val annotations: String? = null,
private val validators: List<ResponseValidator> = emptyList()
) : ContractTest {
override fun testResultRecord(result: Result, response: HttpResponse?): TestResultRecord {
val resultStatus = result.testResult()

val responseStatus = scenario.getStatus(response)
return TestResultRecord(
convertPathParameterStyle(scenario.path),
scenario.method,
responseStatus,
resultStatus,
sourceProvider,
sourceRepository,
sourceRepositoryBranch,
specification,
serviceType
)
}

override fun testDescription(): String {
return scenario.testDescription()
}

override fun runTest(testBaseURL: String, timeOut: Int): Pair<Result, HttpResponse?> {
val log: (LogMessage) -> Unit = { logMessage ->
logger.log(logMessage.withComment(this.annotations))
}

val httpClient = HttpClient(testBaseURL, log = log, timeout = timeOut)

return runTest(httpClient)
}

override fun runTest(testExecutor: TestExecutor): Pair<Result, HttpResponse?> {

val (result, response) = executeTestAndReturnResultAndResponse(scenario, testExecutor, flagsBased)
return Pair(result.updateScenario(scenario), response)
}

override fun plusValidator(validator: ResponseValidator): ScenarioAsTest {
return this.copy(
validators = this.validators.plus(validator)
)
}

private fun logComment() {
if (annotations != null) {
logger.log(annotations)
}
}

private fun executeTestAndReturnResultAndResponse(
testScenario: Scenario,
testExecutor: TestExecutor,
flagsBased: FlagsBased
): Pair<Result, HttpResponse?> {
val request = testScenario.generateHttpRequest(flagsBased)

return try {
testExecutor.setServerState(testScenario.serverState)

testExecutor.preExecuteScenario(testScenario, request)

val response = testExecutor.execute(request)

val validatorResult = validators.asSequence().map { it.validate(scenario, response) }.filterNotNull().firstOrNull()
val result = validatorResult ?: testResult(request, response, testScenario, flagsBased)

Pair(result.withBindings(testScenario.bindings, response), response)
} catch (exception: Throwable) {
Pair(
Result.Failure(exceptionCauseMessage(exception))
.also { failure -> failure.updateScenario(testScenario) }, null)
}
}

private fun testResult(
request: HttpRequest,
response: HttpResponse,
testScenario: Scenario,
flagsBased: FlagsBased? = null
): Result {

val result = when {
response.specmaticResultHeaderValue() == "failure" -> Result.Failure(response.body.toStringLiteral())
.updateScenario(testScenario)
else -> testScenario.matches(request, response, ContractAndResponseMismatch, flagsBased?.unexpectedKeyCheck ?: ValidateUnexpectedKeys)
}

if (result is Result.Success && result.isPartialSuccess()) {
logger.log(" PARTIAL SUCCESS: ${result.partialSuccessMessage}")
logger.newLine()
}

return result
}

}

private fun LogMessage.withComment(comment: String?): LogMessage {
return if (this is HttpLogMessage) {
this.copy(comment = comment)
} else {
this
}
}
Loading

0 comments on commit ca7e622

Please sign in to comment.