misc: add service-level benchmarks

.changes/2fcce0d9-a174-41ab-bb48-f18bbd5a3c5f.json

@@ -0,0 +1,8 @@
+{
+    "id": "2fcce0d9-a174-41ab-bb48-f18bbd5a3c5f",
+    "type": "misc",
+    "description": "Add service-level benchmarks",
+    "issues": [
+        "awslabs/aws-sdk-kotlin#968"
+    ]
+}

codegen/sdk/build.gradle.kts

@@ -37,6 +37,8 @@ tasks["jar"].enabled = false
 fun getProperty(name: String): String? {
     if (project.hasProperty(name)) {
         return project.properties[name].toString()
+    } else if (project.ext.has(name)) {
+        return project.ext[name].toString()
     }
 
     val localProperties = Properties()

gradle.properties

@@ -13,7 +13,7 @@ sdkVersion=0.29.2-SNAPSHOT
 smithyVersion=1.29.0
 smithyGradleVersion=0.6.0
 # smithy-kotlin codegen and runtime are versioned together
-smithyKotlinVersion=0.24.0
+smithyKotlinVersion=0.24.1-SNAPSHOT
 
 # kotlin
 kotlinVersion=1.8.10
@@ -39,4 +39,4 @@ mockkVersion=1.13.3
 slf4jVersion=2.0.6
 
 # dokka config (values specified at build-time as needed)
-smithyKotlinDocBaseUrl=https://sdk.amazonaws.com/kotlin/api/smithy-kotlin/api/$smithyKotlinVersion/
+smithyKotlinDocBaseUrl=https://sdk.amazonaws.com/kotlin/api/smithy-kotlin/api/$smithyKotlinVersion/

settings.gradle.kts

@@ -35,6 +35,7 @@ include(":aws-runtime:aws-config")
 include(":aws-runtime:aws-endpoint")
 include(":aws-runtime:aws-http")
 include(":tests")
+include(":tests:benchmarks:service-benchmarks")
 include(":tests:codegen:event-stream")
 include(":tests:e2e-test-util")
 

tests/benchmarks/service-benchmarks/README.md

@@ -0,0 +1,93 @@
+# Service benchmarks
+
+This module is used for benchmarking the performance of generated clients against AWS services. The top 7 services (by
+traffic coming from the AWS SDK for Kotlin) are tested and metrics are captured with summaries distilled after the runs
+are complete
+
+## Instructions
+
+To run the benchmarks:
+* `./gradlew :tests:benchmarks:service-benchmarks:bootstrapAll`
+  This ensures that all the required service clients are bootstrapped and ready to be built. **You only need to do this
+  once** in your workspace unless you clean up generated services or make a change to codegen.
+* `./gradlew build`
+  This builds the whole SDK.
+* `./gradlew :tests:benchmarks:service-benchmarks:run`
+  This runs the benchmark suite and prints the results to the console formatted as a Markdown table.
+
+## Baseline as of 7/28/2023
+
+The following benchmark run serves as a baseline for future runs:
+
+### Host machine
+
+| Hardware type  | Operating system | Date      |
+|----------------|------------------|-----------|
+| EC2 m5.4xlarge | Amazon Linux 2   | 7/28/2023 |
+
+### Results
+
+|                       | Overhead (ms) |    n |   min |   avg |   med |   p90 |    p99 |    max |
+| :---                  |          ---: | ---: |  ---: |  ---: |  ---: |  ---: |   ---: |   ---: |
+| **S3**                |               |      |       |       |       |       |        |        |
+|   —HeadObject         |               | 1618 | 0.340 | 0.605 | 0.417 | 0.638 |  4.864 | 14.672 |
+|   —PutObject          |               |  766 | 0.310 | 0.557 | 0.392 | 0.675 |  4.008 | 13.358 |
+| **SNS**               |               |      |       |       |       |       |        |        |
+|   —GetTopicAttributes |               | 3458 | 0.233 | 0.514 | 0.373 | 0.515 |  4.378 | 18.719 |
+|   —Publish            |               | 1082 | 0.192 | 0.432 | 0.255 | 0.454 |  3.006 | 19.466 |
+| **STS**               |               |      |       |       |       |       |        |        |
+|   —AssumeRole         |               | 1054 | 0.269 | 0.442 | 0.349 | 0.525 |  0.844 | 19.312 |
+|   —GetCallerIdentity  |               | 4202 | 0.158 | 0.270 | 0.204 | 0.287 |  0.462 | 19.110 |
+| **CloudWatch**        |               |      |       |       |       |       |        |        |
+|   —GetMetricData      |               | 1500 | 0.177 | 1.501 | 0.266 | 5.510 | 13.842 | 18.671 |
+|   —PutMetricData      |               | 2470 | 0.131 | 1.211 | 0.143 | 3.206 | 11.461 | 15.233 |
+| **CloudWatch Events** |               |      |       |       |       |       |        |        |
+|   —DescribeEventBus   |               | 1500 | 0.169 | 0.380 | 0.248 | 0.449 |  3.642 | 11.034 |
+|   —PutEvents          |               | 4007 | 0.159 | 0.340 | 0.210 | 0.344 |  4.881 | 12.941 |
+| **DynamoDB**          |               |      |       |       |       |       |        |        |
+|   —GetItem            |               | 3547 | 0.135 | 0.187 | 0.164 | 0.250 |  0.344 |  4.114 |
+|   —PutItem            |               | 2659 | 0.127 | 0.181 | 0.159 | 0.246 |  0.324 |  2.353 |
+| **Pinpoint**          |               |      |       |       |       |       |        |        |
+|   —GetEndpoint        |               |  368 | 0.245 | 0.436 | 0.380 | 0.669 |  0.824 |  1.238 |
+|   —PutEvents          |               |  297 | 0.277 | 0.376 | 0.351 | 0.505 |  0.696 |  0.717 |
+
+## Methodology
+
+This section describes how the benchmarks actually work at a high level:
+
+### Selection criteria
+
+These benchmarks select a handful of services to test against. The selection criterion is the top 7 services by traffic
+coming from the AWS SDK for Kotlin (i.e., not from other SDKs, console, etc.). As of 7/28, those top 7 services are S3,
+SNS, STS, CloudWatch, CloudWatch Events, DynamoDB, and Pinpoint (in descending order).
+
+For each service, two APIs are selected roughly corresponding to a read and a write operation (e.g., S3::HeadObject is
+a read operation and S3::PutObject is a write operation). Efforts are made to ensure that the APIs selected are the top
+operations by traffic but alternate APIs may be selected in the case of low throttling limits, high setup complexity,
+etc.
+
+### Workflow
+
+Benchmarks are run sequentially in a single thread. This is the high-level workflow for the benchmarks:
+
+* For each benchmark service:
+  * Instantiate a client with a [special telemetry provider](#telemetry-provider)
+  * Run any necessary service-specific setup procedures (e.g., create/configure prerequisite resources)
+  * For each benchmark operation:
+    * Run any necessary operation-specific setup procedures (e.g., create/configure prerequisite resources)
+    * Warmup the API call
+    * Measure the API call
+    * Aggregate operation metrics
+    * Run any necessary operation-specific cleanup procedures (e.g., delete resources created in the setup step)
+  * Run any necessary service-specific cleanup procedures (e.g., delete resources created in the setup step)
+  * Print overall metrics summary
+
+### Telemetry provider
+
+A custom [benchmark-specific telemetry provider][1] is used to instrument each service client. This provider solely
+handles metrics (i.e., no logging, tracing, etc.). It captures specific histogram metrics from an allowlist (currently
+only `smithy.client.attempt_overhead_duration`) and aggregates them for the duration of an operation run (not including
+the warmup phase). After the run is complete, the metrics are aggregated and various statistics are calculated (e.g.,
+minimum, average, median, etc.).
+
+[1]: common/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt

tests/benchmarks/service-benchmarks/build.gradle.kts

@@ -0,0 +1,104 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+buildscript {
+    repositories {
+        mavenCentral()
+    }
+
+    val atomicFuVersion: String by project
+
+    dependencies {
+        classpath("org.jetbrains.kotlinx:atomicfu-gradle-plugin:$atomicFuVersion")
+    }
+}
+
+plugins {
+    kotlin("multiplatform")
+    application
+}
+
+application {
+    mainClass.set("aws.sdk.kotlin.benchmarks.service.BenchmarkHarnessKt")
+}
+
+extra.set("skipPublish", true)
+
+val platforms = listOf("common", "jvm")
+
+platforms.forEach { platform ->
+    apply(from = rootProject.file("gradle/$platform.gradle"))
+}
+
+val requiredServices = setOf(
+    // Top 7 services called by Kotlin SDK customers as of 7/25/2023
+    "s3",
+    "sns",
+    "sts",
+    "cloudwatch",
+    "cloudwatchevents",
+    "dynamodb",
+    "pinpoint",
+
+    // Services required as prerequisites for setup
+    "iam", // Create roles for SNS::AssumeRole
+)
+
+val missingServices = requiredServices.filterNot { rootProject.file("services/$it/build.gradle.kts").exists() }
+
+if (missingServices.isEmpty()) {
+    val optinAnnotations = listOf("kotlin.RequiresOptIn", "aws.smithy.kotlin.runtime.InternalApi")
+
+    kotlin {
+        sourceSets {
+            all {
+                val srcDir = if (name.endsWith("Main")) "src" else "test"
+                val resourcesPrefix = if (name.endsWith("Test")) "test-" else ""
+                // the name is always the platform followed by a suffix of either "Main" or "Test" (e.g. jvmMain, commonTest, etc)
+                val platform = name.substring(0, name.length - 4)
+                kotlin.srcDir("$platform/$srcDir")
+                resources.srcDir("$platform/${resourcesPrefix}resources")
+                languageSettings.progressiveMode = true
+                optinAnnotations.forEach { languageSettings.optIn(it) }
+            }
+
+            val atomicFuVersion: String by project
+            val coroutinesVersion: String by project
+            val smithyKotlinVersion: String by project
+
+            commonMain {
+                dependencies {
+                    api("aws.smithy.kotlin:runtime-core:$smithyKotlinVersion")
+                    implementation(project(":aws-runtime:aws-core"))
+                    implementation("org.jetbrains.kotlinx:atomicfu:$atomicFuVersion")
+                    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
+
+                    requiredServices.forEach { implementation(project(":services:$it")) }
+                }
+            }
+        }
+    }
+} else {
+    logger.warn(
+        "Skipping build for {} project, missing the following services: {}. To ensure this project builds, run the " +
+            "{}:bootstrapAll task.",
+        project.name,
+        missingServices.joinToString(", "),
+        project.path,
+    )
+}
+
+tasks.register("bootstrapAll") {
+    val bootstrapArg = requiredServices.joinToString(",") { "+$it" }
+    val bootstrapProj = project(":codegen:sdk")
+    bootstrapProj.ext.set("aws.services", bootstrapArg)
+    dependsOn(":codegen:sdk:bootstrap")
+}
+
+tasks.named<JavaExec>("run") {
+    classpath += objects.fileCollection().from(
+        tasks.named("compileKotlinJvm"),
+        configurations.named("jvmRuntimeClasspath"),
+    )
+}

tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt

@@ -0,0 +1,116 @@
+package aws.sdk.kotlin.benchmarks.service
+
+import aws.sdk.kotlin.benchmarks.service.definitions.*
+import aws.sdk.kotlin.benchmarks.service.telemetry.MetricSummary
+import aws.smithy.kotlin.runtime.client.SdkClient
+import aws.smithy.kotlin.runtime.io.use
+import kotlin.time.Duration.Companion.seconds
+import kotlin.time.ExperimentalTime
+import kotlin.time.TimeSource
+
+val DEFAULT_WARMUP_TIME = 5.seconds
+val DEFAULT_ITERATION_TIME = 15.seconds
+
+private val benchmarks = setOf(
+    S3Benchmark(),
+    SnsBenchmark(),
+    StsBenchmark(),
+    CloudwatchBenchmark(),
+    CloudwatchEventsBenchmark(),
+    DynamoDbBenchmark(),
+    PinpointBenchmark(),
+).map {
+    @Suppress("UNCHECKED_CAST")
+    it as ServiceBenchmark<SdkClient>
+}
+
+suspend fun main() {
+    val harness = BenchmarkHarness()
+    harness.execute()
+}
+
+class BenchmarkHarness {
+    private val summaries = mutableMapOf<String, MutableMap<String, Map<String, MetricSummary>>>()
+
+    suspend fun execute() {
+        benchmarks.forEach { execute(it) }
+        println()
+        printResults()
+    }
+
+    private suspend fun execute(benchmark: ServiceBenchmark<SdkClient>) {
+        benchmark.client().use { client ->
+            println("${client.config.clientName}:")
+
+            println("  Setting up...")
+            benchmark.setup(client)
+
+            try {
+                benchmark.operations.forEach { execute(it, client) }
+            } finally {
+                benchmark.tearDown(client)
+            }
+        }
+        println()
+    }
+
+    private suspend fun execute(operation: OperationBenchmark<SdkClient>, client: SdkClient) {
+        println("  ${operation.name}:")
+
+        println("    Setting up...")
+        operation.setup(client)
+
+        try {
+            println("    Warming up for ${operation.warmupMode.explanation}...")
+            forAtLeast(operation.warmupMode) {
+                operation.transact(client)
+            }
+
+            Common.metricAggregator.clear()
+
+            println("    Measuring for ${operation.iterationMode.explanation}...")
+            forAtLeast(operation.iterationMode) {
+                operation.transact(client)
+            }
+
+            val summary = Common.metricAggregator.summarizeAndClear()
+            summaries.getOrPut(client.config.clientName, ::mutableMapOf)[operation.name] = summary
+        } finally {
+            println("    Tearing down...")
+            operation.tearDown(client)
+        }
+    }
+
+    private fun printResults() {
+        val table = ResultsTable.from(summaries)
+        println(table)
+    }
+}
+
+@OptIn(ExperimentalTime::class)
+private inline fun forAtLeast(runMode: RunMode, block: () -> Unit) {
+    val start = TimeSource.Monotonic.markNow()
+
+    when (runMode) {
+        is RunMode.Time -> {
+            var cnt = 0
+            while (start.elapsedNow() < runMode.time) {
+                block()
+                cnt++
+            }
+            println("      (completed $cnt iterations)")
+        }
+
+        is RunMode.Iterations -> {
+            repeat(runMode.iterations) {
+                block()
+            }
+            println("      (took ${start.elapsedNow()})")
+        }
+    }
+}
+
+private val RunMode.explanation get() = when (this) {
+    is RunMode.Iterations -> "$iterations iterations"
+    is RunMode.Time -> time.toString()
+}

tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/Common.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package aws.sdk.kotlin.benchmarks.service
+
+import aws.sdk.kotlin.benchmarks.service.telemetry.BenchmarkTelemetryProvider
+import aws.sdk.kotlin.benchmarks.service.telemetry.MetricAggregator
+import aws.smithy.kotlin.runtime.ExperimentalApi
+import aws.smithy.kotlin.runtime.retries.StandardRetryStrategy
+import aws.smithy.kotlin.runtime.util.Uuid
+
+object Common {
+    val metricAggregator = MetricAggregator()
+
+    val noRetries = StandardRetryStrategy {
+        maxAttempts = 1
+    }
+
+    @OptIn(ExperimentalApi::class)
+    val telemetryProvider = BenchmarkTelemetryProvider(metricAggregator)
+
+    fun random(prefix: String = "") = "$prefix${Uuid.random()}"
+}