From 69337901fa05ab5ce969e789883f0bad1a7670ab Mon Sep 17 00:00:00 2001 From: 0marperez <60363173+0marperez@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:10:18 -0700 Subject: [PATCH] feat: aws smoke tests support (#1437) --- .gitignore | 5 +- codegen/aws-sdk-codegen/build.gradle.kts | 1 + .../aws/sdk/kotlin/codegen/GradleGenerator.kt | 70 ++++++++ .../model/traits/testing/SmokeTestTraits.kt | 27 +++ ...AwsSmokeTestsRunnerGeneratorIntegration.kt | 160 ++++++++++++++++++ .../SmokeTestsDenyListIntegration.kt | 37 ---- .../SmokeTestFailHttpEngineIntegration.kt | 50 ++++++ .../SmokeTestSuccessHttpEngineIntegration.kt | 37 ++++ ...tlin.codegen.integration.KotlinIntegration | 4 +- gradle/libs.versions.toml | 4 +- settings.gradle.kts | 9 + tests/codegen/smoke-tests/build.gradle.kts | 126 ++++++++++++++ .../smoke-tests/services/build.gradle.kts | 36 ++++ .../src/test/kotlin/SmokeTestE2ETest.kt | 68 ++++++++ .../resources/smoke-tests-exception.smithy | 53 ++++++ .../test/resources/smoke-tests-failure.smithy | 71 ++++++++ .../test/resources/smoke-tests-success.smithy | 88 ++++++++++ 17 files changed, 805 insertions(+), 41 deletions(-) create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/model/traits/testing/SmokeTestTraits.kt create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/AwsSmokeTestsRunnerGeneratorIntegration.kt delete mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/SmokeTestsDenyListIntegration.kt create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/testing/SmokeTestFailHttpEngineIntegration.kt create mode 100644 codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/testing/SmokeTestSuccessHttpEngineIntegration.kt create mode 100644 tests/codegen/smoke-tests/build.gradle.kts create mode 100644 tests/codegen/smoke-tests/services/build.gradle.kts create mode 100644 tests/codegen/smoke-tests/src/test/kotlin/SmokeTestE2ETest.kt create mode 100644 tests/codegen/smoke-tests/src/test/resources/smoke-tests-exception.smithy create mode 100644 tests/codegen/smoke-tests/src/test/resources/smoke-tests-failure.smithy create mode 100644 tests/codegen/smoke-tests/src/test/resources/smoke-tests-success.smithy diff --git a/.gitignore b/.gitignore index 740eefaf436..c6ae9dc5eb6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,11 @@ build/ .idea/ __pycache__/ local.properties + # ignore generated files services/*/generated-src services/*/build.gradle.kts .kotest/ -*.klib \ No newline at end of file +*.klib +tests/codegen/smoke-tests/services/*/generated-src +tests/codegen/smoke-tests/services/*/build.gradle.kts \ No newline at end of file diff --git a/codegen/aws-sdk-codegen/build.gradle.kts b/codegen/aws-sdk-codegen/build.gradle.kts index 719643eabc8..aa404d7a4cb 100644 --- a/codegen/aws-sdk-codegen/build.gradle.kts +++ b/codegen/aws-sdk-codegen/build.gradle.kts @@ -29,6 +29,7 @@ dependencies { api(libs.smithy.aws.cloudformation.traits) api(libs.smithy.protocol.test.traits) implementation(libs.smithy.aws.endpoints) + implementation(libs.smithy.smoke.test.traits) testImplementation(libs.junit.jupiter) testImplementation(libs.junit.jupiter.params) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/GradleGenerator.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/GradleGenerator.kt index 0cae160ab7b..f590cf1a790 100644 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/GradleGenerator.kt +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/GradleGenerator.kt @@ -4,9 +4,16 @@ */ package aws.sdk.kotlin.codegen +import aws.sdk.kotlin.codegen.model.traits.testing.TestFailedResponseTrait +import aws.sdk.kotlin.codegen.model.traits.testing.TestSuccessResponseTrait import software.amazon.smithy.kotlin.codegen.core.* import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +import software.amazon.smithy.kotlin.codegen.model.expectShape +import software.amazon.smithy.kotlin.codegen.model.hasTrait import software.amazon.smithy.kotlin.codegen.rendering.GradleWriter +import software.amazon.smithy.kotlin.codegen.utils.topDownOperations +import software.amazon.smithy.model.shapes.ServiceShape +import software.amazon.smithy.smoketests.traits.SmokeTestsTrait // TODO - would be nice to allow integrations to define custom settings in the plugin // e.g. we could then more consistently apply this integration if we could define a property like: `build.isAwsSdk: true` @@ -64,9 +71,72 @@ class GradleGenerator : KotlinIntegration { } } } + if (ctx.model.topDownOperations(ctx.settings.service).any { it.hasTrait() }) { + write("") + generateSmokeTestConfig(writer, ctx) + } } val contents = writer.toString() delegator.fileManifest.writeFile("build.gradle.kts", contents) } + + private fun generateSmokeTestConfig(writer: GradleWriter, ctx: CodegenContext) { + generateSmokeTestJarTask(writer, ctx) + writer.write("") + generateSmokeTestTask(writer, ctx) + } + + /** + * Generates a gradle task to create smoke test runner JARs + */ + private fun generateSmokeTestJarTask(writer: GradleWriter, ctx: CodegenContext) { + writer.withBlock("jvm {", "}") { + withBlock("compilations {", "}") { + write("val mainPath = getByName(#S).output.classesDirs", "main") + write("val testPath = getByName(#S).output.classesDirs", "test") + withBlock("tasks {", "}") { + withBlock("register(#S) {", "}", "smokeTestJar") { + write("description = #S", "Creates smoke tests jar") + write("group = #S", "application") + write("dependsOn(build)") + write("mustRunAfter(build)") + withBlock("manifest {", "}") { + write("attributes[#S] = #S", "Main-Class", "${ctx.settings.pkg.name}.smoketests.SmokeTestsKt") + } + write("val runtimePath = configurations.getByName(#S).map { if (it.isDirectory) it else zipTree(it) }", "jvmRuntimeClasspath") + write("duplicatesStrategy = DuplicatesStrategy.EXCLUDE") + write("from(runtimePath, mainPath, testPath)") + write("archiveBaseName.set(#S)", "\${project.name}-smoketests") + } + } + } + } + } + + /** + * Generates a gradle task to run smoke tests + */ + private fun generateSmokeTestTask(writer: GradleWriter, ctx: CodegenContext) { + val hasSuccessResponseTrait = ctx.model.expectShape(ctx.settings.service).hasTrait(TestSuccessResponseTrait.ID) + val hasFailedResponseTrait = ctx.model.expectShape(ctx.settings.service).hasTrait(TestFailedResponseTrait.ID) + val inTestingEnvironment = hasFailedResponseTrait || hasSuccessResponseTrait + + /** + * E2E tests don't have sdkVersion in jar names. They're added later for publishing. + * @see SmokeTestE2ETest + */ + val jarName = if (inTestingEnvironment) "\${project.name}-smoketests.jar" else "\${project.name}-smoketests-\$sdkVersion.jar" + + writer.withBlock("tasks.register(#S) {", "}", "smokeTest") { + write("description = #S", "Runs smoke tests jar") + write("group = #S", "verification") + write("dependsOn(tasks.getByName(#S))", "smokeTestJar") + write("mustRunAfter(tasks.getByName(#S))", "smokeTestJar") + write("") + write("val sdkVersion: String by project") + write("val jarFile = file(#S)", "build/libs/$jarName") + write("classpath = files(jarFile)") + } + } } diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/model/traits/testing/SmokeTestTraits.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/model/traits/testing/SmokeTestTraits.kt new file mode 100644 index 00000000000..6805c3bad1f --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/model/traits/testing/SmokeTestTraits.kt @@ -0,0 +1,27 @@ +package aws.sdk.kotlin.codegen.model.traits.testing + +import software.amazon.smithy.model.node.ObjectNode +import software.amazon.smithy.model.shapes.ShapeId +import software.amazon.smithy.model.traits.AnnotationTrait + +/** + * Indicates the annotated service should always return a failed response. + * IMPORTANT: This trait is intended for use in integration or E2E tests only, not in real-life smoke tests that run + * against a service endpoint. + */ +class TestFailedResponseTrait(node: ObjectNode) : AnnotationTrait(ID, node) { + companion object { + val ID: ShapeId = ShapeId.from("smithy.kotlin.traits#failedResponseTrait") + } +} + +/** + * Indicates the annotated service should always return a success response. + * IMPORTANT: This trait is intended for use in integration or E2E tests only, not in real-life smoke tests that run + * against a service endpoint. + */ +class TestSuccessResponseTrait(node: ObjectNode) : AnnotationTrait(ID, node) { + companion object { + val ID: ShapeId = ShapeId.from("smithy.kotlin.traits#successResponseTrait") + } +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/AwsSmokeTestsRunnerGeneratorIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/AwsSmokeTestsRunnerGeneratorIntegration.kt new file mode 100644 index 00000000000..1e694ae4bbc --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/AwsSmokeTestsRunnerGeneratorIntegration.kt @@ -0,0 +1,160 @@ +package aws.sdk.kotlin.codegen.smoketests + +import aws.sdk.kotlin.codegen.AwsRuntimeTypes +import software.amazon.smithy.kotlin.codegen.KotlinSettings +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes +import software.amazon.smithy.kotlin.codegen.core.getContextValue +import software.amazon.smithy.kotlin.codegen.core.withBlock +import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding +import software.amazon.smithy.kotlin.codegen.model.hasTrait +import software.amazon.smithy.kotlin.codegen.rendering.smoketests.* +import software.amazon.smithy.kotlin.codegen.utils.topDownOperations +import software.amazon.smithy.model.Model +import software.amazon.smithy.smoketests.traits.SmokeTestsTrait + +/** + * Generates AWS specific code for smoke test runners + */ +class AwsSmokeTestsRunnerGeneratorIntegration : KotlinIntegration { + override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = + model.topDownOperations(settings.service).any { it.hasTrait() } + + override val sectionWriters: List + get() = listOf( + AwsSmokeTestsRunnerGenerator.regionEnvironmentVariable, + AwsSmokeTestsRunnerGenerator.clientConfig, + AwsSmokeTestsRunnerGenerator.defaultClientConfig, + AwsSmokeTestsRunnerGenerator.skipTagsEnvironmentVariable, + AwsSmokeTestsRunnerGenerator.serviceFilterEnvironmentVariable, + ) +} + +/** + * The section writer bindings used by [AwsSmokeTestsRunnerGeneratorIntegration] + */ +private object AwsSmokeTestsRunnerGenerator { + /** + * Adds region environment variable support to AWS smoke test runners. + * Preserves other environment variables added via section writer binding, if any. + */ + val regionEnvironmentVariable = + SectionWriterBinding(SmokeTestSectionIds.AdditionalEnvironmentVariables) { writer, previous -> + writer.write("#L", previous) + writer.write( + "private val regionOverride = #T.System.getenv(#S)", + RuntimeTypes.Core.Utils.PlatformProvider, + AWS_REGION, + ) + } + + /** + * Add AWS specific client config support to AWS smoke test runners + */ + val clientConfig = + SectionWriterBinding(SmokeTestSectionIds.ClientConfig) { writer, _ -> + val name = writer.getContextValue(SmokeTestSectionIds.ClientConfig.Name) + val value = writer.getContextValue(SmokeTestSectionIds.ClientConfig.Value) + + // Normalize client config names + val newName = when (name) { + "uri" -> "endpointProvider" + "useDualstack" -> "useDualStack" + "sigv4aRegionSet" -> "sigV4aSigningRegionSet" + "useAccountIdRouting" -> "accountIdEndpointMode" + "useAccelerate" -> "enableAccelerate" + "useMultiRegionAccessPoints" -> "disableMrap" + "useGlobalEndpoint" -> { + writer.write("throw #T(#S)", RuntimeTypes.Core.SmokeTests.SmokeTestsException, "'useGlobalEndpoint' is not supported by the SDK") + return@SectionWriterBinding + } + else -> name + } + writer.writeInline("#L = ", newName) + + // Normalize client values + when (newName) { + "endpointProvider" -> { + val endpointProvider = writer.getContextValue(SmokeTestSectionIds.ClientConfig.EndpointProvider) + val endpointParameters = writer.getContextValue(SmokeTestSectionIds.ClientConfig.EndpointParams) + + writer.withBlock("object : #T {", "}", endpointProvider) { + write( + "override suspend fun resolveEndpoint(params: #1T): #2T = #2T(#3L)", + endpointParameters, + RuntimeTypes.SmithyClient.Endpoints.Endpoint, + value, + ) + } + } + "sigV4aSigningRegionSet" -> { + // Render new value + writer.write("#L.toSet()", value) + // Also configure sigV4a - TODO: Remove once sigV4a is supported for default signer. + writer.write( + "authSchemes = listOf(#T(#T))", + RuntimeTypes.Auth.HttpAuthAws.SigV4AsymmetricAuthScheme, + RuntimeTypes.Auth.AwsSigningCrt.CrtAwsSigner, + ) + } + "accountIdEndpointMode" -> { + when (value) { + "true" -> writer.write("#T.REQUIRED", AwsRuntimeTypes.Config.Endpoints.AccountIdEndpointMode) + "false" -> writer.write("#T.DISABLED", AwsRuntimeTypes.Config.Endpoints.AccountIdEndpointMode) + } + } + "disableMrap" -> { + when (value) { + "true" -> writer.write("false") + "false" -> writer.write("true") + } + } + "region" -> { + writer.write("regionOverride ?: #L", value) + } + else -> writer.write("#L", value) + } + } + + /** + * Add default client config to AWS smoke test runners. + * Preserves previous default config if any. + */ + val defaultClientConfig = + SectionWriterBinding(SmokeTestSectionIds.DefaultClientConfig) { writer, previous -> + writer.write("#L", previous) + writer.write("region = regionOverride") + } + + /** + * Replaces environment variable with one specific to AWS smoke test runners + */ + val skipTagsEnvironmentVariable = + SectionWriterBinding(SmokeTestSectionIds.SkipTags) { writer, _ -> writer.writeInline("#S", AWS_SKIP_TAGS) } + + /** + * Replaces environment variable with one specific to AWS smoke test runners + */ + val serviceFilterEnvironmentVariable = + SectionWriterBinding(SmokeTestSectionIds.ServiceFilter) { writer, _ -> writer.writeInline("#S", AWS_SERVICE_FILTER) } +} + +/** + * Env var for AWS smoke test runners. + * Should be a string that corresponds to an AWS region. + * The region to use when executing smoke tests. This value MUST override any value supplied in the smoke tests themselves. + */ +private const val AWS_REGION = "AWS_SMOKE_TEST_REGION" + +/** + * Env var for AWS smoke test runners. + * Should be a comma-delimited list of strings that correspond to tags on the test cases. + * If a test case is tagged with one of the tags indicated by AWS_SMOKE_TEST_SKIP_TAGS, it MUST be skipped by the smoke test runner. + */ +const val AWS_SKIP_TAGS = "AWS_SMOKE_TEST_SKIP_TAGS" + +/** + * Env var for AWS smoke test runners. + * Should be a comma-separated list of service identifiers to test. + */ +const val AWS_SERVICE_FILTER = "AWS_SMOKE_TEST_SERVICE_IDS" diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/SmokeTestsDenyListIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/SmokeTestsDenyListIntegration.kt deleted file mode 100644 index 66d5196b2f3..00000000000 --- a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/SmokeTestsDenyListIntegration.kt +++ /dev/null @@ -1,37 +0,0 @@ -package aws.sdk.kotlin.codegen.smoketests - -import software.amazon.smithy.kotlin.codegen.KotlinSettings -import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration -import software.amazon.smithy.kotlin.codegen.integration.SectionWriter -import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding -import software.amazon.smithy.kotlin.codegen.rendering.smoketests.SmokeTestsRunner -import software.amazon.smithy.model.Model - -/** - * Will wipe the smoke test runner file for services that are deny listed. - * - * Some services model smoke tests incorrectly and the code generated file will not compile. - */ -class SmokeTestsDenyListIntegration : KotlinIntegration { - override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = - settings.sdkId in smokeTestDenyList - - override val sectionWriters: List - get() = listOf( - SectionWriterBinding(SmokeTestsRunner, smokeTestDenyListSectionWriter), - ) - - private val smokeTestDenyListSectionWriter = SectionWriter { writer, _ -> - writer.write("// Smoke tests for service deny listed until model is fixed") - } -} - -/** - * SDK ID's of services that model smoke tests incorrectly - */ -val smokeTestDenyList = setOf( - "Application Auto Scaling", - "SWF", - "WAFV2", - "IoT Data Plane", -) diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/testing/SmokeTestFailHttpEngineIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/testing/SmokeTestFailHttpEngineIntegration.kt new file mode 100644 index 00000000000..ad6bd0ee95e --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/testing/SmokeTestFailHttpEngineIntegration.kt @@ -0,0 +1,50 @@ +package aws.sdk.kotlin.codegen.smoketests.testing + +import aws.sdk.kotlin.codegen.model.traits.testing.TestFailedResponseTrait +import aws.sdk.kotlin.codegen.model.traits.testing.TestSuccessResponseTrait +import software.amazon.smithy.kotlin.codegen.KotlinSettings +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes +import software.amazon.smithy.kotlin.codegen.core.withBlock +import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +import software.amazon.smithy.kotlin.codegen.integration.SectionWriter +import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding +import software.amazon.smithy.kotlin.codegen.model.expectShape +import software.amazon.smithy.kotlin.codegen.model.hasTrait +import software.amazon.smithy.kotlin.codegen.rendering.smoketests.SmokeTestSectionIds +import software.amazon.smithy.kotlin.codegen.utils.topDownOperations +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.ServiceShape +import software.amazon.smithy.smoketests.traits.SmokeTestsTrait + +/** + * Adds [TestFailedResponseTrait] support to smoke tests + * IMPORTANT: This integration is intended for use in integration or E2E tests only, not in real-life smoke tests that run + * against a service endpoint. + */ +class SmokeTestFailHttpEngineIntegration : KotlinIntegration { + override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = + model.topDownOperations(settings.service).any { it.hasTrait() } && + !model.expectShape(settings.service).hasTrait(TestSuccessResponseTrait.ID) && + model.expectShape(settings.service).hasTrait(TestFailedResponseTrait.ID) + + override val sectionWriters: List + get() = listOf( + SectionWriterBinding(SmokeTestSectionIds.HttpEngineOverride, httpClientOverride), + ) + + private val httpClientOverride = SectionWriter { writer, _ -> + writer.withBlock("httpClient = #T(", ")", RuntimeTypes.HttpTest.TestEngine) { + withBlock("roundTripImpl = { _, request ->", "}") { + write( + "val resp = #T(#T.BadRequest, #T.Empty, #T.Empty)", + RuntimeTypes.Http.Response.HttpResponse, + RuntimeTypes.Http.StatusCode, + RuntimeTypes.Http.Headers, + RuntimeTypes.Http.HttpBody, + ) + write("val now = #T.now()", RuntimeTypes.Core.Instant) + write("#T(request, resp, now, now)", RuntimeTypes.Http.HttpCall) + } + } + } +} diff --git a/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/testing/SmokeTestSuccessHttpEngineIntegration.kt b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/testing/SmokeTestSuccessHttpEngineIntegration.kt new file mode 100644 index 00000000000..03077a4cd5a --- /dev/null +++ b/codegen/aws-sdk-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/smoketests/testing/SmokeTestSuccessHttpEngineIntegration.kt @@ -0,0 +1,37 @@ +package aws.sdk.kotlin.codegen.smoketests.testing + +import aws.sdk.kotlin.codegen.model.traits.testing.TestFailedResponseTrait +import aws.sdk.kotlin.codegen.model.traits.testing.TestSuccessResponseTrait +import software.amazon.smithy.kotlin.codegen.KotlinSettings +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes +import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +import software.amazon.smithy.kotlin.codegen.integration.SectionWriter +import software.amazon.smithy.kotlin.codegen.integration.SectionWriterBinding +import software.amazon.smithy.kotlin.codegen.model.expectShape +import software.amazon.smithy.kotlin.codegen.model.hasTrait +import software.amazon.smithy.kotlin.codegen.rendering.smoketests.SmokeTestSectionIds +import software.amazon.smithy.kotlin.codegen.utils.topDownOperations +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.ServiceShape +import software.amazon.smithy.smoketests.traits.SmokeTestsTrait + +/** + * Adds [TestSuccessResponseTrait] support to smoke tests + * IMPORTANT: This integration is intended for use in integration or E2E tests only, not in real-life smoke tests that run + * against a service endpoint. + */ +class SmokeTestSuccessHttpEngineIntegration : KotlinIntegration { + override fun enabledForService(model: Model, settings: KotlinSettings): Boolean = + model.topDownOperations(settings.service).any { it.hasTrait() } && + model.expectShape(settings.service).hasTrait(TestSuccessResponseTrait.ID) && + !model.expectShape(settings.service).hasTrait(TestFailedResponseTrait.ID) + + override val sectionWriters: List + get() = listOf( + SectionWriterBinding(SmokeTestSectionIds.HttpEngineOverride, httpClientOverride), + ) + + private val httpClientOverride = SectionWriter { writer, _ -> + writer.write("httpClient = #T()", RuntimeTypes.HttpTest.TestEngine) + } +} diff --git a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration index 6d8199d3072..de1ab6cd4ac 100644 --- a/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration +++ b/codegen/aws-sdk-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration @@ -42,5 +42,7 @@ aws.sdk.kotlin.codegen.customization.s3.express.SigV4S3ExpressAuthSchemeIntegrat aws.sdk.kotlin.codegen.customization.s3.express.S3ExpressIntegration aws.sdk.kotlin.codegen.customization.s3.S3ExpiresIntegration aws.sdk.kotlin.codegen.BusinessMetricsIntegration -aws.sdk.kotlin.codegen.smoketests.SmokeTestsDenyListIntegration +aws.sdk.kotlin.codegen.smoketests.AwsSmokeTestsRunnerGeneratorIntegration +aws.sdk.kotlin.codegen.smoketests.testing.SmokeTestSuccessHttpEngineIntegration +aws.sdk.kotlin.codegen.smoketests.testing.SmokeTestFailHttpEngineIntegration aws.sdk.kotlin.codegen.customization.AwsQueryModeCustomization diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d86529cab72..d003593d62b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,8 +11,8 @@ coroutines-version = "1.9.0" atomicfu-version = "0.25.0" # smithy-kotlin codegen and runtime are versioned separately -smithy-kotlin-runtime-version = "1.3.19" -smithy-kotlin-codegen-version = "0.33.19" +smithy-kotlin-runtime-version = "1.3.20" +smithy-kotlin-codegen-version = "0.33.20" # codegen smithy-version = "1.51.0" diff --git a/settings.gradle.kts b/settings.gradle.kts index 203fcef7b8d..fa43e4d2789 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -54,6 +54,8 @@ include(":services") include(":tests") include(":tests:codegen:event-stream") include(":tests:e2e-test-util") +include(":tests:codegen:smoke-tests") +include(":tests:codegen:smoke-tests:services") // generated services val File.isServiceDir: Boolean @@ -68,6 +70,13 @@ file("services").listFiles().forEach { } } +// generated services by smoke tests test suite +file("tests/codegen/smoke-tests/services").listFiles().forEach { + if (it.isServiceDir) { + include(":tests:codegen:smoke-tests:services:${it.name}") + } +} + if ("dynamodb".isBootstrappedService) { include(":hll:dynamodb-mapper") include(":hll:dynamodb-mapper:dynamodb-mapper") diff --git a/tests/codegen/smoke-tests/build.gradle.kts b/tests/codegen/smoke-tests/build.gradle.kts new file mode 100644 index 00000000000..77a74afceba --- /dev/null +++ b/tests/codegen/smoke-tests/build.gradle.kts @@ -0,0 +1,126 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +import aws.sdk.kotlin.gradle.codegen.dsl.generateSmithyProjections +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import aws.sdk.kotlin.gradle.codegen.smithyKotlinProjectionPath + +description = "Tests for smoke tests runners" + +plugins { + alias(libs.plugins.aws.kotlin.repo.tools.smithybuild) + alias(libs.plugins.kotlin.jvm) +} + +val projections = listOf( + Projection("successService", "smoke-tests-success.smithy", "smithy.kotlin.traits#SuccessService"), + Projection("failureService", "smoke-tests-failure.smithy", "smithy.kotlin.traits#FailureService"), + Projection("exceptionService", "smoke-tests-exception.smithy", "smithy.kotlin.traits#ExceptionService"), +) + +configureProject() +configureProjections() +configureTasks() + +fun configureProject() { + val codegen by configurations.getting + + dependencies { + codegen(project(":codegen:aws-sdk-codegen")) + codegen(libs.smithy.cli) + codegen(libs.smithy.model) + + implementation(project(":codegen:aws-sdk-codegen")) + implementation(libs.smithy.kotlin.codegen) + + testImplementation(libs.kotlin.test) + testImplementation(gradleTestKit()) + } +} + +fun configureProjections() { + smithyBuild { + val pathToSmithyModels = "src/test/resources/" + + this@Build_gradle.projections.forEach { projection -> + projections.register(projection.name) { + imports = listOf(layout.projectDirectory.file(pathToSmithyModels + projection.modelFile).asFile.absolutePath) + smithyKotlinPlugin { + serviceShapeId = projection.serviceShapeId + packageName = "aws.sdk.kotlin.test" + packageVersion = "1.0" + buildSettings { + generateFullProject = false + generateDefaultBuildFiles = false + optInAnnotations = listOf( + "aws.smithy.kotlin.runtime.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", + ) + } + } + } + } + } + + tasks.withType { + dependsOn(tasks.generateSmithyProjections) + kotlinOptions.allWarningsAsErrors = false + } +} + +fun configureTasks() { + tasks.register("stageServices") { + dependsOn(tasks.generateSmithyProjections) + + doLast { + this@Build_gradle.projections.forEach { projection -> + val projectionPath = smithyBuild.smithyKotlinProjectionPath(projection.name).get() + val destinationPath = layout.projectDirectory.asFile.absolutePath + "/services/${projection.name}" + + copy { + from("$projectionPath/src") + into("$destinationPath/generated-src") + } + copy { + from("$projectionPath/build.gradle.kts") + into(destinationPath) + } + } + } + } + + tasks.build { + dependsOn(tasks.getByName("stageServices")) + mustRunAfter(tasks.getByName("stageServices")) + } + + tasks.clean { + this@Build_gradle.projections.forEach { projection -> + delete("services/${projection.name}") + } + } + + tasks.withType { + dependsOn(tasks.getByName("stageServices")) + mustRunAfter(tasks.getByName("stageServices")) + + testLogging { + events("passed", "skipped", "failed") + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + showExceptions = true + showCauses = true + showStackTraces = true + } + } +} + +/** + * Holds metadata about a smithy projection + */ +data class Projection( + val name: String, + val modelFile: String, + val serviceShapeId: String, +) diff --git a/tests/codegen/smoke-tests/services/build.gradle.kts b/tests/codegen/smoke-tests/services/build.gradle.kts new file mode 100644 index 00000000000..2238ac4b0d1 --- /dev/null +++ b/tests/codegen/smoke-tests/services/build.gradle.kts @@ -0,0 +1,36 @@ +import aws.sdk.kotlin.gradle.kmp.kotlin + +plugins { + alias(libs.plugins.aws.kotlin.repo.tools.kmp) apply false +} + +// capture locally - scope issue with custom KMP plugin +val libraries = libs + +subprojects { + apply { + plugin(libraries.plugins.kotlin.multiplatform.get().pluginId) + plugin(libraries.plugins.aws.kotlin.repo.tools.kmp.get().pluginId) + } + + kotlin { + + jvm() + + sourceSets { + all { + languageSettings.optIn("kotlin.RequiresOptIn") + languageSettings.optIn("aws.smithy.kotlin.runtime.InternalApi") + languageSettings.optIn("aws.sdk.kotlin.runtime.InternalSdkApi") + } + + commonMain { + kotlin.srcDir("generated-src/main/kotlin") + } + + commonTest { + kotlin.srcDir("generated-src/test/kotlin") + } + } + } +} diff --git a/tests/codegen/smoke-tests/src/test/kotlin/SmokeTestE2ETest.kt b/tests/codegen/smoke-tests/src/test/kotlin/SmokeTestE2ETest.kt new file mode 100644 index 00000000000..61f38fe62c1 --- /dev/null +++ b/tests/codegen/smoke-tests/src/test/kotlin/SmokeTestE2ETest.kt @@ -0,0 +1,68 @@ +import aws.sdk.kotlin.codegen.smoketests.AWS_SERVICE_FILTER +import aws.sdk.kotlin.codegen.smoketests.AWS_SKIP_TAGS +import org.gradle.testkit.runner.GradleRunner +import java.io.File +import kotlin.test.* + +class SmokeTestE2ETest { + @Test + fun successService() { + val smokeTestRunnerOutput = runSmokeTests("successService") + + assertContains(smokeTestRunnerOutput, "ok SuccessService SuccessTest - no error expected from service") + assertContains(smokeTestRunnerOutput, "ok SuccessService SuccessTestWithTags - no error expected from service") + } + + @Test + fun failureService() { + val smokeTestRunnerOutput = runSmokeTests("failureService") + + assertContains(smokeTestRunnerOutput, "ok FailureService FailuresTest - error expected from service") + } + + @Test + fun exceptionService() { + val smokeTestRunnerOutput = runSmokeTests("exceptionService", expectingFailure = true) + + assertContains(smokeTestRunnerOutput, "not ok ExceptionService ExceptionTest - no error expected from service") + assertContains(smokeTestRunnerOutput, "#aws.smithy.kotlin.runtime.http.interceptors.SmokeTestsFailureException: Smoke test failed with HTTP status code: 400") + assertContains(smokeTestRunnerOutput, "#\tat aws.smithy.kotlin.runtime.http.interceptors.SmokeTestsInterceptor.readBeforeDeserialization(SmokeTestsInterceptor.kt:19)") + assertContains(smokeTestRunnerOutput, "#\tat aws.smithy.kotlin.runtime.http.interceptors.InterceptorExecutor.readBeforeDeserialization(InterceptorExecutor.kt:252)") + } + + @Test + fun successServiceSkipTags() { + val envVars = mapOf(AWS_SKIP_TAGS to "success") + val smokeTestRunnerOutput = runSmokeTests("successService", envVars) + + assertContains(smokeTestRunnerOutput, "ok SuccessService SuccessTest - no error expected from service") + assertContains(smokeTestRunnerOutput, "ok SuccessService SuccessTestWithTags - no error expected from service # skip") + } + + @Test + fun successServiceServiceFilter() { + val envVars = mapOf(AWS_SERVICE_FILTER to "Failure") // Only run tests for services with this SDK ID + val smokeTestRunnerOutput = runSmokeTests("successService", envVars) + + assertContains(smokeTestRunnerOutput, "ok SuccessService SuccessTest - no error expected from service # skip") + assertContains(smokeTestRunnerOutput, "ok SuccessService SuccessTestWithTags - no error expected from service # skip") + } +} + +private fun runSmokeTests( + service: String, + envVars: Map = emptyMap(), + expectingFailure: Boolean = false, +): String { + val sdkRootDir = System.getProperty("user.dir") + "/../../../" + + val task = GradleRunner.create() + .withProjectDir(File(sdkRootDir)) + // FIXME: Remove `-Paws.kotlin.native=false` when Kotlin Native is ready + .withArguments("-Paws.kotlin.native=false", ":tests:codegen:smoke-tests:services:$service:smokeTest") + .withEnvironment(envVars) + + val buildResult = if (expectingFailure) task.buildAndFail() else task.build() + + return buildResult.output +} diff --git a/tests/codegen/smoke-tests/src/test/resources/smoke-tests-exception.smithy b/tests/codegen/smoke-tests/src/test/resources/smoke-tests-exception.smithy new file mode 100644 index 00000000000..24bc4f7183b --- /dev/null +++ b/tests/codegen/smoke-tests/src/test/resources/smoke-tests-exception.smithy @@ -0,0 +1,53 @@ +$version: "2" +namespace smithy.kotlin.traits + +use aws.protocols#awsJson1_0 +use aws.api#service +use smithy.test#smokeTests +use smithy.rules#endpointRuleSet + +@trait(selector: "service") +structure failedResponseTrait { } + +@failedResponseTrait +@awsJson1_0 +@service(sdkId: "Exception") +@endpointRuleSet( + version: "1.0", + parameters: {}, + rules: [ + { + "type": "endpoint", + "conditions": [], + "endpoint": { + "url": "https://static.endpoint" + } + } + ] +) +service ExceptionService { + version: "1.0.0", + operations: [ TestOperation ], +} + +@smokeTests( + [ + { + id: "ExceptionTest" + expect: { + success: {} + } + } + ] +) +operation TestOperation { + input := { + bar: String + } + errors: [ + InvalidMessageError + ] +} + +@error("client") +structure InvalidMessageError {} diff --git a/tests/codegen/smoke-tests/src/test/resources/smoke-tests-failure.smithy b/tests/codegen/smoke-tests/src/test/resources/smoke-tests-failure.smithy new file mode 100644 index 00000000000..05f1b8563dc --- /dev/null +++ b/tests/codegen/smoke-tests/src/test/resources/smoke-tests-failure.smithy @@ -0,0 +1,71 @@ +$version: "2" +namespace smithy.kotlin.traits + +use aws.protocols#awsJson1_0 +use aws.api#service +use smithy.test#smokeTests +use smithy.rules#endpointRuleSet + +@trait(selector: "service") +structure failedResponseTrait { } + +@failedResponseTrait +@awsJson1_0 +@service(sdkId: "Failure") +@endpointRuleSet( + version: "1.0", + parameters: {}, + rules: [ + { + "type": "endpoint", + "conditions": [], + "endpoint": { + "url": "https://static.endpoint" + } + } + ] +) +service FailureService { + version: "1.0.0", + operations: [ TestOperation ], +} + +@smokeTests( + [ + { + id: "FailuresTest" + params: {bar: "2"} + expect: { + failure: {} + } + vendorParamsShape: AwsVendorParams, + vendorParams: { + region: "eu-central-1" + uri: "https://failure.amazonaws.com" + useFips: false + useDualstack: false + } + } + ] +) +operation TestOperation { + input := { + bar: String + } + errors: [ + InvalidMessageError + ] +} + +@error("client") +structure InvalidMessageError {} + +@mixin +structure BaseAwsVendorParams { + region: String = "us-west-2" + uri: String + useFips: Boolean = false + useDualstack: Boolean = false +} + +structure AwsVendorParams with [BaseAwsVendorParams] {} \ No newline at end of file diff --git a/tests/codegen/smoke-tests/src/test/resources/smoke-tests-success.smithy b/tests/codegen/smoke-tests/src/test/resources/smoke-tests-success.smithy new file mode 100644 index 00000000000..4f27498fc2e --- /dev/null +++ b/tests/codegen/smoke-tests/src/test/resources/smoke-tests-success.smithy @@ -0,0 +1,88 @@ +$version: "2" +namespace smithy.kotlin.traits + +use aws.protocols#awsJson1_0 +use aws.api#service +use smithy.test#smokeTests +use smithy.rules#endpointRuleSet + +@trait(selector: "service") +structure successResponseTrait { } + +@successResponseTrait +@awsJson1_0 +@service(sdkId: "Success") +@endpointRuleSet( + version: "1.0", + parameters: {}, + rules: [ + { + "type": "endpoint", + "conditions": [], + "endpoint": { + "url": "https://static.endpoint" + } + } + ] +) +service SuccessService { + version: "1.0.0", + operations: [ TestOperation ], +} + +@smokeTests( + [ + { + id: "SuccessTest" + params: {bar: "2"} + expect: { + success: {} + } + vendorParamsShape: AwsVendorParams, + vendorParams: { + region: "eu-central-1" + uri: "https://success.amazonaws.com" + useFips: false + useDualstack: false + } + }, + { + id: "SuccessTestWithTags" + params: {bar: "2"} + tags: [ + "success" + ] + expect: { + success: {} + } + vendorParamsShape: AwsVendorParams, + vendorParams: { + region: "eu-central-1" + uri: "https://success.amazonaws.com" + useFips: false + useDualstack: false + } + } + ] +) +operation TestOperation { + input := { + bar: String + } + errors: [ + InvalidMessageError + ] +} + +@error("client") +structure InvalidMessageError {} + +@mixin +structure BaseAwsVendorParams { + region: String = "us-west-2" + uri: String + useFips: Boolean = false + useDualstack: Boolean = false +} + +structure AwsVendorParams with [BaseAwsVendorParams] {}