diff --git a/.changes/2fcce0d9-a174-41ab-bb48-f18bbd5a3c5f.json b/.changes/2fcce0d9-a174-41ab-bb48-f18bbd5a3c5f.json new file mode 100644 index 00000000000..2f34e15fb43 --- /dev/null +++ b/.changes/2fcce0d9-a174-41ab-bb48-f18bbd5a3c5f.json @@ -0,0 +1,8 @@ +{ + "id": "2fcce0d9-a174-41ab-bb48-f18bbd5a3c5f", + "type": "misc", + "description": "Add service-level benchmarks", + "issues": [ + "awslabs/aws-sdk-kotlin#968" + ] +} diff --git a/codegen/sdk/build.gradle.kts b/codegen/sdk/build.gradle.kts index 45ef4e37511..dce7e884ffb 100644 --- a/codegen/sdk/build.gradle.kts +++ b/codegen/sdk/build.gradle.kts @@ -37,6 +37,8 @@ tasks["jar"].enabled = false fun getProperty(name: String): String? { if (project.hasProperty(name)) { return project.properties[name].toString() + } else if (project.ext.has(name)) { + return project.ext[name].toString() } val localProperties = Properties() diff --git a/settings.gradle.kts b/settings.gradle.kts index 90e654511bf..7dfe77fd3bc 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -35,6 +35,7 @@ include(":aws-runtime:aws-config") include(":aws-runtime:aws-endpoint") include(":aws-runtime:aws-http") include(":tests") +include(":tests:benchmarks:service-benchmarks") include(":tests:codegen:event-stream") include(":tests:e2e-test-util") diff --git a/tests/benchmarks/service-benchmarks/README.md b/tests/benchmarks/service-benchmarks/README.md new file mode 100644 index 00000000000..5d13740fd5e --- /dev/null +++ b/tests/benchmarks/service-benchmarks/README.md @@ -0,0 +1,93 @@ +# Service benchmarks + +This module is used for benchmarking the performance of generated clients against AWS services. The top 7 services (by +traffic coming from the AWS SDK for Kotlin) are tested and metrics are captured with summaries distilled after the runs +are complete + +## Instructions + +To run the benchmarks: +* `./gradlew :tests:benchmarks:service-benchmarks:bootstrapAll` + This ensures that all the required service clients are bootstrapped and ready to be built. **You only need to do this + once** in your workspace unless you clean up generated services or make a change to codegen. +* `./gradlew build` + This builds the whole SDK. +* `./gradlew :tests:benchmarks:service-benchmarks:run` + This runs the benchmark suite and prints the results to the console formatted as a Markdown table. + +## Baseline as of 8/8/2023 + +The following benchmark run serves as a baseline for future runs: + +### Environment + +| Hardware type | Operating system | SDK version | +|----------------|------------------|-----------------| +| EC2 m5.4xlarge | Amazon Linux 2 | 0.30.0-SNAPSHOT | + +### Results + +| | Overhead (ms) | n | min | avg | med | p90 | p99 | max | +| :--- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| **S3** | | | | | | | | | +| —HeadObject | | 1715 | 0.334 | 0.561 | 0.379 | 0.521 | 3.149 | 20.071 | +| —PutObject | | 739 | 0.306 | 0.492 | 0.337 | 0.389 | 7.958 | 16.556 | +| **SNS** | | | | | | | | | +| —GetTopicAttributes | | 3041 | 0.235 | 0.494 | 0.354 | 0.461 | 2.964 | 17.129 | +| —Publish | | 1001 | 0.199 | 0.394 | 0.224 | 0.420 | 1.262 | 16.160 | +| **STS** | | | | | | | | | +| —AssumeRole | | 1081 | 0.273 | 0.419 | 0.349 | 0.485 | 0.622 | 14.781 | +| —GetCallerIdentity | | 4705 | 0.157 | 0.242 | 0.184 | 0.217 | 0.414 | 13.459 | +| **CloudWatch** | | | | | | | | | +| —GetMetricData | | 1500 | 0.174 | 1.352 | 0.219 | 3.239 | 13.830 | 15.193 | +| —PutMetricData | | 2452 | 0.133 | 1.194 | 0.144 | 1.911 | 13.007 | 14.862 | +| **CloudWatch Events** | | | | | | | | | +| —DescribeEventBus | | 1500 | 0.156 | 0.290 | 0.187 | 0.238 | 0.530 | 18.934 | +| —PutEvents | | 4577 | 0.152 | 0.293 | 0.176 | 0.378 | 3.921 | 10.022 | +| **DynamoDB** | | | | | | | | | +| —GetItem | | 4223 | 0.135 | 0.154 | 0.148 | 0.164 | 0.216 | 2.415 | +| —PutItem | | 3059 | 0.130 | 0.154 | 0.145 | 0.178 | 0.193 | 1.771 | +| **Pinpoint** | | | | | | | | | +| —GetEndpoint | | 555 | 0.220 | 0.401 | 0.406 | 0.452 | 0.506 | 6.606 | +| —PutEvents | | 415 | 0.242 | 0.400 | 0.420 | 0.466 | 0.619 | 2.762 | + +## Methodology + +This section describes how the benchmarks actually work at a high level: + +### Selection criteria + +These benchmarks select a handful of services to test against. The selection criterion is the top 7 services by traffic +coming from the AWS SDK for Kotlin (i.e., not from other SDKs, console, etc.). As of 7/28, those top 7 services are S3, +SNS, STS, CloudWatch, CloudWatch Events, DynamoDB, and Pinpoint (in descending order). + +For each service, two APIs are selected roughly corresponding to a read and a write operation (e.g., S3::HeadObject is +a read operation and S3::PutObject is a write operation). Efforts are made to ensure that the APIs selected are the top +operations by traffic but alternate APIs may be selected in the case of low throttling limits, high setup complexity, +etc. + +### Workflow + +Benchmarks are run sequentially in a single thread. This is the high-level workflow for the benchmarks: + +* For each benchmark service: + * Instantiate a client with a [special telemetry provider](#telemetry-provider) + * Run any necessary service-specific setup procedures (e.g., create/configure prerequisite resources) + * For each benchmark operation: + * Run any necessary operation-specific setup procedures (e.g., create/configure prerequisite resources) + * Warmup the API call + * Measure the API call + * Aggregate operation metrics + * Run any necessary operation-specific cleanup procedures (e.g., delete resources created in the setup step) + * Run any necessary service-specific cleanup procedures (e.g., delete resources created in the setup step) + * Print overall metrics summary + +### Telemetry provider + +A custom [benchmark-specific telemetry provider][1] is used to instrument each service client. This provider solely +handles metrics (i.e., no logging, tracing, etc.). It captures specific histogram metrics from an allowlist (currently +only `smithy.client.attempt_overhead_duration`) and aggregates them for the duration of an operation run (not including +the warmup phase). After the run is complete, the metrics are aggregated and various statistics are calculated (e.g., +minimum, average, median, etc.). + +[1]: common/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt diff --git a/tests/benchmarks/service-benchmarks/build.gradle.kts b/tests/benchmarks/service-benchmarks/build.gradle.kts new file mode 100644 index 00000000000..11c80b28988 --- /dev/null +++ b/tests/benchmarks/service-benchmarks/build.gradle.kts @@ -0,0 +1,104 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +buildscript { + repositories { + mavenCentral() + } + + val atomicFuVersion: String by project + + dependencies { + classpath("org.jetbrains.kotlinx:atomicfu-gradle-plugin:$atomicFuVersion") + } +} + +plugins { + kotlin("multiplatform") + application +} + +application { + mainClass.set("aws.sdk.kotlin.benchmarks.service.BenchmarkHarnessKt") +} + +extra.set("skipPublish", true) + +val platforms = listOf("common", "jvm") + +platforms.forEach { platform -> + apply(from = rootProject.file("gradle/$platform.gradle")) +} + +val requiredServices = setOf( + // Top 7 services called by Kotlin SDK customers as of 7/25/2023, in descending order of call volume + "s3", + "sns", + "sts", + "cloudwatch", + "cloudwatchevents", + "dynamodb", + "pinpoint", + + // Services required as prerequisites for setup + "iam", // Create roles for STS::AssumeRole +) + +val missingServices = requiredServices.filterNot { rootProject.file("services/$it/build.gradle.kts").exists() } + +if (missingServices.isEmpty()) { + val optinAnnotations = listOf("kotlin.RequiresOptIn", "aws.smithy.kotlin.runtime.InternalApi") + + kotlin { + sourceSets { + all { + val srcDir = if (name.endsWith("Main")) "src" else "test" + val resourcesPrefix = if (name.endsWith("Test")) "test-" else "" + // the name is always the platform followed by a suffix of either "Main" or "Test" (e.g. jvmMain, commonTest, etc) + val platform = name.substring(0, name.length - 4) + kotlin.srcDir("$platform/$srcDir") + resources.srcDir("$platform/${resourcesPrefix}resources") + languageSettings.progressiveMode = true + optinAnnotations.forEach { languageSettings.optIn(it) } + } + + val atomicFuVersion: String by project + val coroutinesVersion: String by project + val smithyKotlinVersion: String by project + + commonMain { + dependencies { + api("aws.smithy.kotlin:runtime-core:$smithyKotlinVersion") + implementation(project(":aws-runtime:aws-core")) + implementation("org.jetbrains.kotlinx:atomicfu:$atomicFuVersion") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion") + + requiredServices.forEach { implementation(project(":services:$it")) } + } + } + } + } +} else { + logger.warn( + "Skipping build for {} project, missing the following services: {}. To ensure this project builds, run the " + + "{}:bootstrapAll task.", + project.name, + missingServices.joinToString(", "), + project.path, + ) +} + +tasks.register("bootstrapAll") { + val bootstrapArg = requiredServices.joinToString(",") { "+$it" } + val bootstrapProj = project(":codegen:sdk") + bootstrapProj.ext.set("aws.services", bootstrapArg) + dependsOn(":codegen:sdk:bootstrap") +} + +tasks.named("run") { + classpath += objects.fileCollection().from( + tasks.named("compileKotlinJvm"), + configurations.named("jvmRuntimeClasspath"), + ) +} diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt new file mode 100644 index 00000000000..dd0d942802c --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/BenchmarkHarness.kt @@ -0,0 +1,116 @@ +package aws.sdk.kotlin.benchmarks.service + +import aws.sdk.kotlin.benchmarks.service.definitions.* +import aws.sdk.kotlin.benchmarks.service.telemetry.MetricSummary +import aws.smithy.kotlin.runtime.client.SdkClient +import aws.smithy.kotlin.runtime.io.use +import kotlin.time.Duration.Companion.seconds +import kotlin.time.ExperimentalTime +import kotlin.time.TimeSource + +val DEFAULT_WARMUP_TIME = 5.seconds +val DEFAULT_ITERATION_TIME = 15.seconds + +private val benchmarks = setOf( + S3Benchmark(), + SnsBenchmark(), + StsBenchmark(), + CloudwatchBenchmark(), + CloudwatchEventsBenchmark(), + DynamoDbBenchmark(), + PinpointBenchmark(), +).map { + @Suppress("UNCHECKED_CAST") + it as ServiceBenchmark +} + +suspend fun main() { + val harness = BenchmarkHarness() + harness.execute() +} + +class BenchmarkHarness { + private val summaries = mutableMapOf>>() + + suspend fun execute() { + benchmarks.forEach { execute(it) } + println() + printResults() + } + + private suspend fun execute(benchmark: ServiceBenchmark) { + benchmark.client().use { client -> + println("${client.config.clientName}:") + + println(" Setting up...") + benchmark.setup(client) + + try { + benchmark.operations.forEach { execute(it, client) } + } finally { + benchmark.tearDown(client) + } + } + println() + } + + private suspend fun execute(operation: OperationBenchmark, client: SdkClient) { + println(" ${operation.name}:") + + println(" Setting up...") + operation.setup(client) + + try { + println(" Warming up for ${operation.warmupMode.explanation}...") + forAtLeast(operation.warmupMode) { + operation.transact(client) + } + + Common.metricAggregator.clear() + + println(" Measuring for ${operation.iterationMode.explanation}...") + forAtLeast(operation.iterationMode) { + operation.transact(client) + } + + val summary = Common.metricAggregator.summarizeAndClear() + summaries.getOrPut(client.config.clientName, ::mutableMapOf)[operation.name] = summary + } finally { + println(" Tearing down...") + operation.tearDown(client) + } + } + + private fun printResults() { + val table = ResultsTable.from(summaries) + println(table) + } +} + +@OptIn(ExperimentalTime::class) +private inline fun forAtLeast(runMode: RunMode, block: () -> Unit) { + val start = TimeSource.Monotonic.markNow() + + when (runMode) { + is RunMode.Time -> { + var cnt = 0 + while (start.elapsedNow() < runMode.time) { + block() + cnt++ + } + println(" (completed $cnt iterations)") + } + + is RunMode.Iterations -> { + repeat(runMode.iterations) { + block() + } + println(" (took ${start.elapsedNow()})") + } + } +} + +private val RunMode.explanation get() = when (this) { + is RunMode.Iterations -> "$iterations iterations" + is RunMode.Time -> time.toString() +} diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/Common.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/Common.kt new file mode 100644 index 00000000000..125c31b31e6 --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/Common.kt @@ -0,0 +1,24 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.benchmarks.service + +import aws.sdk.kotlin.benchmarks.service.telemetry.BenchmarkTelemetryProvider +import aws.sdk.kotlin.benchmarks.service.telemetry.MetricAggregator +import aws.smithy.kotlin.runtime.ExperimentalApi +import aws.smithy.kotlin.runtime.retries.StandardRetryStrategy +import aws.smithy.kotlin.runtime.util.Uuid + +object Common { + val metricAggregator = MetricAggregator() + + val noRetries = StandardRetryStrategy { + maxAttempts = 1 + } + + @OptIn(ExperimentalApi::class) + val telemetryProvider = BenchmarkTelemetryProvider(metricAggregator) + + fun random(prefix: String = "") = "$prefix${Uuid.random()}" +} diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/ResultsTable.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/ResultsTable.kt new file mode 100644 index 00000000000..8400b2fd047 --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/ResultsTable.kt @@ -0,0 +1,161 @@ +package aws.sdk.kotlin.benchmarks.service + +import aws.sdk.kotlin.benchmarks.service.telemetry.MetricSummary +import kotlin.math.max +import kotlin.math.pow +import kotlin.math.round + +private const val NAME_FIELD = "name" +private const val COUNT_FIELD = "n" + +/** + * A set of results from service benchmarks. The results are keyed first by service, then by operation, then by metric + * name. For instance: + * + * ```json + * { + * "S3": { + * "HeadObject": { + * "Overhead (ms)": { + * "count": 1618, + * "statistics": { + * "min": 0.340, + * "avg": 0.605, + * "med": 0.417, + * ... + * } + * }, + * ... + * }, + * ... + * }, + * ... + * } + * ``` + */ +private typealias Results = Map>> + +class ResultsTable private constructor(private val columns: List, private val rows: List>) { + companion object { + fun from(results: Results): ResultsTable { + val columnMapper = ColumnMapper.from(results) + val rows = RowGenerator(columnMapper).generate(results) + + val maxWidths = Array(columnMapper.count) { 0 } + rows.forEach { row -> + row.forEachIndexed { idx, value -> + maxWidths[idx] = max(maxWidths[idx], value.length) + } + } + + val columns = maxWidths.mapIndexed { index, maxWidth -> + val alignment = if (index < 1) HorizontalAlignment.LEFT else HorizontalAlignment.RIGHT + Column(maxWidth, alignment) + } + + return ResultsTable(columns, rows) + } + } + + override fun toString(): String = buildString { + rows.forEach { row -> + append('|') + row.forEachIndexed { index, cell -> + val column = columns[index] + append(' ') + append(column.alignment.pad(cell, column.maxWidth)) + append(" |") + } + appendLine() + } + } +} + +private enum class HorizontalAlignment(val pad: String.(Int) -> String) { + LEFT(String::padEnd), + RIGHT(String::padStart), +} + +private data class Column(val maxWidth: Int, val alignment: HorizontalAlignment) + +private data class ColumnMapper(val count: Int, val mapping: Map>) { + companion object { + fun from(results: Results): ColumnMapper { + var count = 1 // One for the left-most column holding subject + + val mapping = mutableMapOf>() + results.values.forEach { service -> + service.values.forEach { operation -> + operation.entries.forEach { (metric, summary) -> + val metricMapping = mapping.getOrPut(metric, ::mutableMapOf) + metricMapping.getOrPut(NAME_FIELD) { count++ } + metricMapping.getOrPut(COUNT_FIELD) { count++ } + summary.statistics.keys.forEach { statistic -> + metricMapping.getOrPut(statistic) { count++ } + } + } + } + } + + return ColumnMapper(count, mapping) + } + } +} + +private data class RowGenerator(val columnMapper: ColumnMapper) { + fun generate(results: Results) = buildList { + add(headerRow()) + add(delineatorRow()) + + // Value rows + results.forEach { (service, operations) -> + add(serviceRow(service)) + operations.forEach { (operation, metrics) -> + add(operationRow(operation, metrics)) + } + } + } + + private fun delineatorRow() = row { idx -> if (idx < 1) ":---" else "---:" } + + private fun headerRow(): Array { + val header = row() + columnMapper.mapping.forEach { (metric, metricColumnMapping) -> + metricColumnMapping.keys.forEach { statistic -> + header[metricColumnMapping.getValue(statistic)] = when (statistic) { + NAME_FIELD -> metric + else -> statistic + } + } + } + return header + } + + private fun operationRow(operation: String, metrics: Map): Array { + val row = row() + row[0] = " —$operation" + metrics.forEach { (metric, summary) -> + val metricColumnMapping = columnMapper.mapping.getValue(metric) + row[metricColumnMapping.getValue(COUNT_FIELD)] = summary.count.toString() + summary.statistics.forEach { (statistic, value) -> + row[metricColumnMapping.getValue(statistic)] = value.format() + } + } + return row + } + + private fun serviceRow(service: String) = row { idx -> if (idx == 0) "**$service**" else "" } + + private fun row(init: (Int) -> String = { "" }) = Array(columnMapper.count, init) +} + +private fun Double.format(precision: Int = 3): String { + val magnitude = 10.0.pow(precision) + val default = (round(this * magnitude) / magnitude).toString() + val chunks = default.split(".") + return chunks[0] + "." + if (chunks.size == 1) { + "0".repeat(precision) + } else { + chunks[1].padEnd(precision, '0') + } +} diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/CloudwatchBenchmark.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/CloudwatchBenchmark.kt new file mode 100644 index 00000000000..f7ca62eedad --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/CloudwatchBenchmark.kt @@ -0,0 +1,87 @@ +package aws.sdk.kotlin.benchmarks.service.definitions + +import aws.sdk.kotlin.benchmarks.service.Common +import aws.sdk.kotlin.services.cloudwatch.CloudWatchClient +import aws.sdk.kotlin.services.cloudwatch.getMetricData +import aws.sdk.kotlin.services.cloudwatch.model.MetricDataQuery +import aws.sdk.kotlin.services.cloudwatch.model.MetricDatum +import aws.sdk.kotlin.services.cloudwatch.putMetricData +import aws.smithy.kotlin.runtime.ExperimentalApi +import aws.smithy.kotlin.runtime.time.Instant +import kotlinx.coroutines.delay +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.minutes + +private const val NAMESPACE = "SdkBenchmark/testdata" +private const val METRIC_NAME = "foo" +private const val METRIC_VALUE = 42.0 + +class CloudwatchBenchmark : ServiceBenchmark { + @OptIn(ExperimentalApi::class) + override suspend fun client() = CloudWatchClient.fromEnvironment { + retryStrategy = Common.noRetries + telemetryProvider = Common.telemetryProvider + httpClient { + telemetryProvider = Common.telemetryProvider + } + } + + override val operations get() = listOf(getMetricDataBenchmark, putMetricDataBenchmark) + + private val getMetricDataBenchmark = object : AbstractOperationBenchmark("GetMetricData") { + // Default CloudWatch::GetMetricData max TPS is 50, so we artificially throttle the benchmark. + // Adjust the run mode to ensure we get a minimum amount of transactions. + override val warmupMode = RunMode.Iterations(500) + override val iterationMode = RunMode.Iterations(1500) + + override suspend fun setup(client: CloudWatchClient) { + client.putMetricData { + namespace = NAMESPACE + metricData = listOf( + MetricDatum { + metricName = METRIC_NAME + value = METRIC_VALUE + timestamp = Instant.now() + }, + ) + } + } + + override suspend fun transact(client: CloudWatchClient) { + delay(20.milliseconds) // Default CloudWatch::GetMetricData max TPS is 50 + client.getMetricData { + startTime = Instant.now() - 5.minutes + endTime = Instant.now() + metricDataQueries = listOf( + MetricDataQuery { + id = "fooQuery" + metricStat { + metric { + namespace = NAMESPACE + metricName = METRIC_NAME + } + period = 60 // 60 seconds is the minimum period for regular resolution metrics + stat = "Average" + } + returnData = false // just return the stat, not the full dataset + }, + ) + } + } + } + + private val putMetricDataBenchmark = object : AbstractOperationBenchmark("PutMetricData") { + override suspend fun transact(client: CloudWatchClient) { + client.putMetricData { + namespace = NAMESPACE + metricData = listOf( + MetricDatum { + metricName = METRIC_NAME + value = METRIC_VALUE + timestamp = Instant.now() + }, + ) + } + } + } +} diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/CloudwatchEventsBenchmark.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/CloudwatchEventsBenchmark.kt new file mode 100644 index 00000000000..1a353e34617 --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/CloudwatchEventsBenchmark.kt @@ -0,0 +1,69 @@ +package aws.sdk.kotlin.benchmarks.service.definitions + +import aws.sdk.kotlin.benchmarks.service.Common +import aws.sdk.kotlin.services.cloudwatchevents.* +import aws.sdk.kotlin.services.cloudwatchevents.model.PutEventsRequestEntry +import aws.smithy.kotlin.runtime.ExperimentalApi +import aws.smithy.kotlin.runtime.time.Instant +import kotlinx.coroutines.delay +import kotlin.time.Duration.Companion.milliseconds + +class CloudwatchEventsBenchmark : ServiceBenchmark { + private lateinit var eventBus: String + + @OptIn(ExperimentalApi::class) + override suspend fun client() = CloudWatchEventsClient.fromEnvironment { + retryStrategy = Common.noRetries + telemetryProvider = Common.telemetryProvider + httpClient { + telemetryProvider = Common.telemetryProvider + } + } + + override suspend fun setup(client: CloudWatchEventsClient) { + eventBus = Common.random("sdk-benchmark-eventbus-") + client.createEventBus { + name = eventBus + } + } + + override val operations get() = listOf(describeEventBusBenchmark, putEventsBenchmark) + + override suspend fun tearDown(client: CloudWatchEventsClient) { + client.deleteEventBus { + name = eventBus + } + } + + private val describeEventBusBenchmark = + object : AbstractOperationBenchmark("DescribeEventBus") { + // Default CloudWatchEvents::DescribeEventBus max TPS is 50, so we artificially throttle the benchmark. + // Adjust the run mode to ensure we get a minimum amount of transactions. + override val warmupMode = RunMode.Iterations(500) + override val iterationMode = RunMode.Iterations(1500) + + override suspend fun transact(client: CloudWatchEventsClient) { + delay(20.milliseconds) // Default CloudWatchEvents::DescribeEventBus max TPS is 50 + client.describeEventBus { + name = eventBus + } + } + } + + private val putEventsBenchmark = + object : AbstractOperationBenchmark("PutEvents") { + override suspend fun transact(client: CloudWatchEventsClient) { + client.putEvents { + entries = listOf( + PutEventsRequestEntry { + eventBusName = eventBus + detail = """{ "foo": "bar" }""" + detailType = "foo" + source = "baz" + time = Instant.now() + }, + ) + } + } + } +} diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/DynamoDbBenchmark.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/DynamoDbBenchmark.kt new file mode 100644 index 00000000000..4aadf2b1de7 --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/DynamoDbBenchmark.kt @@ -0,0 +1,85 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.benchmarks.service.definitions + +import aws.sdk.kotlin.benchmarks.service.Common +import aws.sdk.kotlin.services.dynamodb.* +import aws.sdk.kotlin.services.dynamodb.model.* +import aws.sdk.kotlin.services.dynamodb.waiters.waitUntilTableExists +import aws.smithy.kotlin.runtime.ExperimentalApi + +class DynamoDbBenchmark : ServiceBenchmark { + private val table = Common.random("sdk-benchmark-table-") + + @OptIn(ExperimentalApi::class) + override suspend fun client() = DynamoDbClient.fromEnvironment { + retryStrategy = Common.noRetries + telemetryProvider = Common.telemetryProvider + httpClient { + telemetryProvider = Common.telemetryProvider + } + } + + override suspend fun setup(client: DynamoDbClient) { + client.createTable { + tableName = table + billingMode = BillingMode.PayPerRequest + attributeDefinitions = listOf( + AttributeDefinition { + attributeName = "id" + attributeType = ScalarAttributeType.S + }, + ) + keySchema = listOf( + KeySchemaElement { + attributeName = "id" + keyType = KeyType.Hash + }, + ) + } + client.waitUntilTableExists { tableName = table } + } + + override val operations get() = listOf(getItemBenchmark, putItemBenchmark) + + override suspend fun tearDown(client: DynamoDbClient) { + client.deleteTable { tableName = table } + } + + private val getItemBenchmark = object : AbstractOperationBenchmark("GetItem") { + private val knownId = randomAttr() + + override suspend fun setup(client: DynamoDbClient) { + client.putItem { + tableName = table + item = mapOf( + "id" to knownId, + "value" to randomAttr(), + ) + } + } + + override suspend fun transact(client: DynamoDbClient) { + client.getItem { + tableName = table + key = mapOf("id" to knownId) + } + } + } + + private val putItemBenchmark = object : AbstractOperationBenchmark("PutItem") { + override suspend fun transact(client: DynamoDbClient) { + client.putItem { + tableName = table + item = mapOf( + "id" to randomAttr(), + "value" to randomAttr(), + ) + } + } + } +} + +private fun randomAttr() = AttributeValue.S(Common.random()) diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/PinpointBenchmark.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/PinpointBenchmark.kt new file mode 100644 index 00000000000..426d6f4a696 --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/PinpointBenchmark.kt @@ -0,0 +1,89 @@ +package aws.sdk.kotlin.benchmarks.service.definitions + +import aws.sdk.kotlin.benchmarks.service.Common +import aws.sdk.kotlin.services.pinpoint.* +import aws.sdk.kotlin.services.pinpoint.model.ChannelType +import aws.sdk.kotlin.services.pinpoint.model.Event +import aws.sdk.kotlin.services.pinpoint.model.EventsBatch +import aws.smithy.kotlin.runtime.ExperimentalApi +import aws.smithy.kotlin.runtime.time.Instant + +class PinpointBenchmark : ServiceBenchmark { + private val epAddress = Common.random("sdk-benchmark-address-") + private lateinit var appId: String + private val epId = Common.random("sdk-benchmark-endpoint-") + + @OptIn(ExperimentalApi::class) + override suspend fun client() = PinpointClient.fromEnvironment { + retryStrategy = Common.noRetries + telemetryProvider = Common.telemetryProvider + httpClient { + telemetryProvider = Common.telemetryProvider + } + } + + override suspend fun setup(client: PinpointClient) { + val resp = client.createApp { + createApplicationRequest { + name = Common.random("sdk-benchmark-app-") + } + } + + appId = resp.applicationResponse!!.id!! + + client.updateEndpoint { + applicationId = appId + endpointId = epId + endpointRequest { + address = epAddress + channelType = ChannelType.InApp + } + } + } + + override val operations get() = listOf(getEndpointBenchmark, putEventsBenchmark) + + override suspend fun tearDown(client: PinpointClient) { + client.deleteEndpoint { + applicationId = appId + endpointId = epId + } + + client.deleteApp { + applicationId = appId + } + } + + private val getEndpointBenchmark = object : AbstractOperationBenchmark("GetEndpoint") { + override suspend fun transact(client: PinpointClient) { + client.getEndpoint { + applicationId = appId + endpointId = epId + } + } + } + + private val putEventsBenchmark = object : AbstractOperationBenchmark("PutEvents") { + override suspend fun transact(client: PinpointClient) { + client.putEvents { + applicationId = appId + eventsRequest { + batchItem = mapOf( + Common.random() to EventsBatch { + endpoint { + address = epAddress + } + events = mapOf( + "foo" to Event { + eventType = "Bar" + timestamp = Instant.now().toString() + attributes = mapOf("baz" to "qux") + }, + ) + }, + ) + } + } + } + } +} diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/S3Benchmark.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/S3Benchmark.kt new file mode 100644 index 00000000000..7e349ce3072 --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/S3Benchmark.kt @@ -0,0 +1,85 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.benchmarks.service.definitions + +import aws.sdk.kotlin.benchmarks.service.Common +import aws.sdk.kotlin.services.s3.* +import aws.sdk.kotlin.services.s3.model.BucketLocationConstraint +import aws.smithy.kotlin.runtime.ExperimentalApi +import aws.smithy.kotlin.runtime.content.ByteStream + +class S3Benchmark : ServiceBenchmark { + private val bucketName = Common.random("sdk-benchmark-bucket-") + + companion object { + private const val KEY = "test-object" + private const val CONTENTS = "test-contents" + } + + @OptIn(ExperimentalApi::class) + override suspend fun client() = S3Client.fromEnvironment { + retryStrategy = Common.noRetries + telemetryProvider = Common.telemetryProvider + httpClient { + telemetryProvider = Common.telemetryProvider + } + } + + override suspend fun setup(client: S3Client) { + client.createBucket { + bucket = bucketName + createBucketConfiguration { + locationConstraint = BucketLocationConstraint.fromValue(client.config.region!!) + } + } + } + + override val operations get() = listOf(headObjectBenchmark, putObjectBenchmark) + + override suspend fun tearDown(client: S3Client) { + client.deleteBucket { bucket = bucketName } + } + + private val headObjectBenchmark = object : AbstractOperationBenchmark("HeadObject") { + override suspend fun setup(client: S3Client) { + client.putObject { + bucket = bucketName + key = KEY + body = ByteStream.fromString(CONTENTS) + } + } + + override suspend fun transact(client: S3Client) { + client.headObject { + bucket = bucketName + key = KEY + } + } + + override suspend fun tearDown(client: S3Client) { + client.deleteObject { + bucket = bucketName + key = KEY + } + } + } + + private val putObjectBenchmark = object : AbstractOperationBenchmark("PutObject") { + override suspend fun transact(client: S3Client) { + client.putObject { + bucket = bucketName + key = KEY + body = ByteStream.fromString(CONTENTS) + } + } + + override suspend fun tearDown(client: S3Client) { + client.deleteObject { + bucket = bucketName + key = KEY + } + } + } +} diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/ServiceBenchmark.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/ServiceBenchmark.kt new file mode 100644 index 00000000000..25ac490a12d --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/ServiceBenchmark.kt @@ -0,0 +1,115 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package aws.sdk.kotlin.benchmarks.service.definitions + +import aws.sdk.kotlin.benchmarks.service.DEFAULT_ITERATION_TIME +import aws.sdk.kotlin.benchmarks.service.DEFAULT_WARMUP_TIME +import aws.smithy.kotlin.runtime.client.SdkClient +import kotlin.time.Duration + +/** + * Defines the harness for conducting a benchmark of a service client. + * @param C The type of the specific service client (e.g., [aws.sdk.kotlin.services.s3.S3Client]). + */ +interface ServiceBenchmark { + /** + * Return a configured service client. This method **MUST NOT** perform any additional service setup. + * @return A configured service client. + */ + suspend fun client(): C + + /** + * Sets up a service for benchmarking. This may involve creating/configuring specific resources (e.g., creating an + * S3 bucket into which the benchmarks will read/write objects). Resources created/modified by this method + * **SHOULD** be removed/restored by [tearDown]. + * + * The default implementation of this method does nothing. + * @param client The service client to use. + */ + suspend fun setup(client: C) { } + + /** + * The list of operations to benchmark. + */ + val operations: List> + + /** + * Cleans up a service after benchmarking. This may involve deleting specific resources (e.g., removing an S3 bucket + * which was created for the purpose of benchmarking). This method will be called regardless of whether an exception + * occurred during benchmarking. + * + * The default implementation of this method does nothing. + * @param client The service client to use. + */ + suspend fun tearDown(client: C) { } +} + +/** + * Defines the harness for conducting a benchmark of a specific service operation. + */ +interface OperationBenchmark { + /** + * The name of the operation (e.g., `HeadBucket`). + */ + val name: String + + /** + * The [RunMode] to use while warming up. The default is `RunMode.Time(DEFAULT_WARMUP_TIME)`. + */ + val warmupMode: RunMode get() = RunMode.Time(DEFAULT_WARMUP_TIME) + + /** + * The [RunMode] to use while iterating on the actual benchmark. The default is + * `RunMode.Time(DEFAULT_ITERATION_TIME)`. + */ + val iterationMode: RunMode get() = RunMode.Time(DEFAULT_ITERATION_TIME) + + /** + * Sets up an operation for benchmarking. This may involve creating/configuring specific resources (e.g., creating + * an IAM role for use in an STS AssumeRole benchmark). Resources created/modified by this method **SHOULD** be + * removed/restored by [tearDown]. + * @param client The service client to use. + */ + suspend fun setup(client: C) { } + + /** + * Perform a single service operation. This method **SHOULD** only perform a single service call and perform + * minimal/no validation. This method will be called repeatedly during the warmup and iteration phase. + * @param client The service client to use. + */ + suspend fun transact(client: C) + + /** + * Cleans up an operation after benchmarking. This may involve deleting specific resources (e.g., removing an IAM + * role). This method will be called regardless of whether an exception occurred during benchmarking. + * + * The default implementation of this method does nothing. + * @param client The service client to use. + */ + suspend fun tearDown(client: C) { } +} + +/** + * Identifies the mode in which to run a phase of benchmark execution. + */ +sealed interface RunMode { + /** + * Run for a specific number of iterations. + * @param iterations The number of iterations to run. + */ + data class Iterations(val iterations: Int) : RunMode + + /** + * Run for a specific amount of time. + * @param time The amount of time to run. + */ + data class Time(val time: Duration) : RunMode +} + +/** + * An abstract base class for operation benchmarks. + * @param The name of the operation (e.g., `HeadBucket`). + */ +abstract class AbstractOperationBenchmark(override val name: String) : OperationBenchmark diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/SnsBenchmark.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/SnsBenchmark.kt new file mode 100644 index 00000000000..2edefea0e87 --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/SnsBenchmark.kt @@ -0,0 +1,48 @@ +package aws.sdk.kotlin.benchmarks.service.definitions + +import aws.sdk.kotlin.benchmarks.service.Common +import aws.sdk.kotlin.services.sns.* +import aws.smithy.kotlin.runtime.ExperimentalApi + +class SnsBenchmark : ServiceBenchmark { + private lateinit var arn: String + + @OptIn(ExperimentalApi::class) + override suspend fun client() = SnsClient.fromEnvironment { + retryStrategy = Common.noRetries + telemetryProvider = Common.telemetryProvider + httpClient { + telemetryProvider = Common.telemetryProvider + } + } + + override suspend fun setup(client: SnsClient) { + arn = client.createTopic { + name = Common.random("sdk-benchmark-topic-") + attributes = mapOf("DisplayName" to "Foo") + }.topicArn!! + } + + override val operations get() = listOf(getTopicAttributesBenchmark, publishBenchmark) + + override suspend fun tearDown(client: SnsClient) { + client.deleteTopic { topicArn = arn } + } + + private val getTopicAttributesBenchmark = object : AbstractOperationBenchmark("GetTopicAttributes") { + override suspend fun transact(client: SnsClient) { + client.getTopicAttributes { + topicArn = arn + } + } + } + + private val publishBenchmark = object : AbstractOperationBenchmark("Publish") { + override suspend fun transact(client: SnsClient) { + client.publish { + topicArn = arn + message = Common.random() + } + } + } +} diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/StsBenchmark.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/StsBenchmark.kt new file mode 100644 index 00000000000..141274ea609 --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/definitions/StsBenchmark.kt @@ -0,0 +1,121 @@ +package aws.sdk.kotlin.benchmarks.service.definitions + +import aws.sdk.kotlin.benchmarks.service.Common +import aws.sdk.kotlin.services.iam.IamClient +import aws.sdk.kotlin.services.iam.createRole +import aws.sdk.kotlin.services.iam.deleteRole +import aws.sdk.kotlin.services.sts.StsClient +import aws.sdk.kotlin.services.sts.assumeRole +import aws.sdk.kotlin.services.sts.model.StsException +import aws.smithy.kotlin.runtime.ExperimentalApi +import aws.smithy.kotlin.runtime.io.use +import aws.smithy.kotlin.runtime.telemetry.TelemetryProvider +import kotlinx.coroutines.delay +import kotlinx.coroutines.withTimeout +import kotlin.time.Duration.Companion.seconds + +class StsBenchmark : ServiceBenchmark { + @OptIn(ExperimentalApi::class) + override suspend fun client() = StsClient.fromEnvironment { + retryStrategy = Common.noRetries + telemetryProvider = Common.telemetryProvider + httpClient { + telemetryProvider = Common.telemetryProvider + } + } + + override val operations get() = listOf(assumeRoleBenchmark, getCallerIdentityBenchmark) + + private val assumeRoleBenchmark = object : AbstractOperationBenchmark("AssumeRole") { + private lateinit var iamRoleArn: String + private lateinit var iamRoleName: String + + override suspend fun setup(client: StsClient) { + iamRoleName = Common.random("sdk-benchmark-role-") + val callerArn = client.getCallerIdentity().arn!! + + iam { + val resp = createRole { + roleName = iamRoleName + assumeRolePolicyDocument = assumeRolePolicyJson(callerArn) + } + + iamRoleArn = resp.role!!.arn!! + } + + // It takes a while for newly-created roles to fully propagate to STS. In the meantime, trying to assume the + // role causes an exception. Example: + // | StsException: User: arn:aws:iam::123456789012:user/Username is not authorized to perform: + // | sts:AssumeRole on resource: arn:aws:iam::123456789012:role/RoleName + // Even after a single STS::AssumeRole success, subsequent calls _may still be unsuccessful_. Thus, we ping + // every second until we get 3 successful calls –or– an unexpected error is thrown. + withTimeout(30.seconds) { + var successes = 0 + while (successes < 3) { + try { + delay(1.seconds) + client.assumeRole { + roleArn = iamRoleArn + roleSessionName = Common.random("sdk-benchmark-session-") + } + } catch (e: StsException) { + if (e.isNotAuthorized) { + // Role still being propagated to STS + continue + } else { + // Some other error we didn't expect, throw hands up + throw e + } + } + + // STS successfully assumed role, chalk it up in the win column. + successes++ + } + + // Saw enough successes, we're good to go. + } + } + + override suspend fun transact(client: StsClient) { + client.assumeRole { + roleArn = iamRoleArn + roleSessionName = Common.random("sdk-benchmark-session-") + } + } + + override suspend fun tearDown(client: StsClient) { + iam { deleteRole { roleName = iamRoleName } } + } + } + + private val getCallerIdentityBenchmark = object : AbstractOperationBenchmark("GetCallerIdentity") { + override suspend fun transact(client: StsClient) { + client.getCallerIdentity() + } + } +} + +private suspend inline fun iam(block: IamClient.() -> T) = IamClient + .fromEnvironment { telemetryProvider = TelemetryProvider.None } + .use(block) + +private val StsException.isNotAuthorized: Boolean + get() = message?.contains("is not authorized to perform") == true + +private fun assumeRolePolicyJson(principalArn: String) = // language=JSON + """ + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": [ "$principalArn" ] + }, + "Action": [ + "sts:AssumeRole" + ] + } + ] + } + """.trimIndent() diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt new file mode 100644 index 00000000000..afa83f7e12c --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/telemetry/BenchmarkTelemetryProvider.kt @@ -0,0 +1,110 @@ +package aws.sdk.kotlin.benchmarks.service.telemetry + +import aws.smithy.kotlin.runtime.ExperimentalApi +import aws.smithy.kotlin.runtime.telemetry.TelemetryProvider +import aws.smithy.kotlin.runtime.telemetry.context.Context +import aws.smithy.kotlin.runtime.telemetry.context.ContextManager +import aws.smithy.kotlin.runtime.telemetry.logging.LoggerProvider +import aws.smithy.kotlin.runtime.telemetry.metrics.* +import aws.smithy.kotlin.runtime.telemetry.trace.TracerProvider +import aws.smithy.kotlin.runtime.util.Attributes + +private val capturedMetrics = mapOf( + "smithy.client.attempt_overhead_duration" to "Overhead", + // "smithy.client.http.time_to_first_byte" to "TTFB", + // "smithy.client.attempt_duration" to "Call", + // "smithy.client.serialization_duration" to "Serlz", + // "smithy.client.deserialization_duration" to "Deserlz", + // "smithy.client.resolve_endpoint_duration" to "EPR", +) + +@ExperimentalApi +class BenchmarkTelemetryProvider(private val metricAggregator: MetricAggregator) : TelemetryProvider { + override val contextManager = ContextManager.None + override val loggerProvider = LoggerProvider.None + override val tracerProvider = TracerProvider.None + + override val meterProvider = object : MeterProvider { + override fun getOrCreateMeter(scope: String) = object : Meter { + override fun createUpDownCounter(name: String, units: String?, description: String?) = + NoOpUpDownCounter + + override fun createAsyncUpDownCounter( + name: String, + callback: LongUpDownCounterCallback, + units: String?, + description: String?, + ) = NoOpAsyncMeasurementHandle + + override fun createMonotonicCounter(name: String, units: String?, description: String?) = + NoOpMonotonicCounter + + override fun createLongHistogram(name: String, units: String?, description: String?) = + NoOpLongHistogram + + override fun createDoubleHistogram(name: String, units: String?, description: String?) = + capturedMetrics[name]?.let { BenchmarkDoubleHistogram(it, units) } ?: NoOpDoubleHistogram + + override fun createLongGauge( + name: String, + callback: LongGaugeCallback, + units: String?, + description: String?, + ) = NoOpAsyncMeasurementHandle + + override fun createDoubleGauge( + name: String, + callback: DoubleGaugeCallback, + units: String?, + description: String?, + ) = NoOpAsyncMeasurementHandle + } + } + + private inner class BenchmarkDoubleHistogram(name: String, units: String?) : DoubleHistogram { + private val newUnit: String? + private val transform: (Double) -> Double + + init { + when (units) { + "s" -> { + newUnit = "ms" + transform = { it * 1000 } + } + + null -> { + newUnit = null + transform = { it } + } + + else -> throw IllegalArgumentException("Unknown unit type $units") + } + } + + private val formattedName = name + (newUnit?.let { " ($it)" } ?: "") + + override fun record(value: Double, attributes: Attributes, context: Context?) { + metricAggregator.add(formattedName, transform(value)) + } + } +} + +private object NoOpAsyncMeasurementHandle : AsyncMeasurementHandle { + override fun stop() { } +} + +private object NoOpDoubleHistogram : DoubleHistogram { + override fun record(value: Double, attributes: Attributes, context: Context?) { } +} + +private object NoOpLongHistogram : LongHistogram { + override fun record(value: Long, attributes: Attributes, context: Context?) { } +} + +private object NoOpMonotonicCounter : MonotonicCounter { + override fun add(value: Long, attributes: Attributes, context: Context?) { } +} + +private object NoOpUpDownCounter : UpDownCounter { + override fun add(value: Long, attributes: Attributes, context: Context?) { } +} diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/telemetry/ConcurrentListBuilder.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/telemetry/ConcurrentListBuilder.kt new file mode 100644 index 00000000000..5c5c824988c --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/telemetry/ConcurrentListBuilder.kt @@ -0,0 +1,24 @@ +package aws.sdk.kotlin.benchmarks.service.telemetry + +import kotlinx.atomicfu.atomic +import kotlinx.atomicfu.update + +class ConcurrentListBuilder { + private val head = atomic?>(null) + + fun add(value: T) { + head.update { Node(value, it) } + } + + fun toList(): List { + var ptr: Node? = head.value + return buildList { + while (ptr != null) { + add(ptr!!.value) + ptr = ptr!!.next + } + }.reversed() + } + + private data class Node(val value: T, val next: Node?) +} diff --git a/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/telemetry/MetricAggregator.kt b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/telemetry/MetricAggregator.kt new file mode 100644 index 00000000000..95df7fa83c1 --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/src/aws/sdk/kotlin/benchmarks/service/telemetry/MetricAggregator.kt @@ -0,0 +1,52 @@ +package aws.sdk.kotlin.benchmarks.service.telemetry + +import kotlin.math.roundToInt + +private const val P_SCALE = 1000 + +class MetricAggregator { + private var builder = ConcurrentListBuilder() + + fun add(name: String, value: Double) = builder.add(Metric(name, value)) + + fun clear() { + builder = ConcurrentListBuilder() + } + + fun summarizeAndClear(): Map { + val metrics = builder.toList() + clear() + return metrics + .groupBy(Metric::name, Metric::value) + .mapValues { (_, values) -> MetricSummary(values) } + } + + private data class Metric(val name: String, val value: Double) +} + +data class MetricSummary(val count: Int, val statistics: Map) { + constructor(values: List) : this(values.size, values.summarize()) +} + +private fun List.summarize() = buildMap { + val values = sorted() + put("min", values.first()) + put("avg", values.average()) + put("med", values p 0.5) + put("p90", values p 0.9) + put("p99", values p 0.99) + put("max", values.last()) +} + +infix fun List.p(percentile: Double): Double { + val k = (P_SCALE * percentile * (size - 1)).roundToInt() + val leftIdx = k / P_SCALE + return when (val mod = k.mod(P_SCALE)) { + 0 -> this[leftIdx] + else -> { + val rightScale = mod.toDouble() / P_SCALE + val leftScale = 1 - rightScale + this[leftIdx] * leftScale + this[leftIdx + 1] * rightScale + } + } +} diff --git a/tests/benchmarks/service-benchmarks/common/test/aws/sdk/kotlin/benchmarks/service/telemetry/ConcurrentListBuilderTest.kt b/tests/benchmarks/service-benchmarks/common/test/aws/sdk/kotlin/benchmarks/service/telemetry/ConcurrentListBuilderTest.kt new file mode 100644 index 00000000000..c268c6daf3e --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/test/aws/sdk/kotlin/benchmarks/service/telemetry/ConcurrentListBuilderTest.kt @@ -0,0 +1,26 @@ +package aws.sdk.kotlin.benchmarks.service.telemetry + +import kotlin.test.Test +import kotlin.test.assertEquals + +class ConcurrentListBuilderTest { + @Test + fun testEmpty() { + val builder = ConcurrentListBuilder() + assertEquals(listOf(), builder.toList()) + } + + @Test + fun testNonEmpty() { + val builder = ConcurrentListBuilder() + builder.add("a") + builder.add("b") + builder.add("c") + assertEquals(listOf("a", "b", "c"), builder.toList()) + + builder.add("d") + builder.add("e") + builder.add("f") + assertEquals(listOf("a", "b", "c", "d", "e", "f"), builder.toList()) + } +} diff --git a/tests/benchmarks/service-benchmarks/common/test/aws/sdk/kotlin/benchmarks/service/telemetry/MetricAggregatorTest.kt b/tests/benchmarks/service-benchmarks/common/test/aws/sdk/kotlin/benchmarks/service/telemetry/MetricAggregatorTest.kt new file mode 100644 index 00000000000..36ce5a863cd --- /dev/null +++ b/tests/benchmarks/service-benchmarks/common/test/aws/sdk/kotlin/benchmarks/service/telemetry/MetricAggregatorTest.kt @@ -0,0 +1,53 @@ +package aws.sdk.kotlin.benchmarks.service.telemetry + +import org.junit.jupiter.api.Test +import kotlin.math.max +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +private const val TOLERANCE = 0.005 + +class MetricAggregatorTest { + @Test + fun testSummarization() { + val fibbonacis = listOf(1, 2, 3, 0, 1, 8, 5).map(Int::toDouble) + val primes = listOf(23, 37, 5, 41, 31, 3, 17, 59, 11, 2, 19, 29, 7, 47, 43, 13, 53).map(Int::toDouble) + val aggregator = MetricAggregator() + + // Interleave metric values + (0..max(fibbonacis.size, primes.size)).forEach { idx -> + if (idx < fibbonacis.size) aggregator.add("fibbonaci", fibbonacis[idx]) + if (idx < primes.size) aggregator.add("prime", primes[idx]) + } + + val summary = aggregator.summarizeAndClear() + + assertEquals(2, summary.size) + + val fibbonaciSummary = assertNotNull(summary["fibbonaci"]) + assertEquals(7, fibbonaciSummary.count) + assertStats(0.0, 2.86, 2.0, 6.2, 7.82, 8.0, fibbonaciSummary.statistics) + + val primeSummary = assertNotNull(summary["prime"]) + assertEquals(17, primeSummary.count) + assertStats(2.0, 25.88, 23.0, 49.4, 58.04, 59.0, primeSummary.statistics) + } + + private fun assertStats( + min: Double, + avg: Double, + med: Double, + p90: Double, + p99: Double, + max: Double, + stats: Map, + ) { + assertEquals(6, stats.size) + assertEquals(min, stats["min"]!!, TOLERANCE) + assertEquals(avg, stats["avg"]!!, TOLERANCE) + assertEquals(med, stats["med"]!!, TOLERANCE) + assertEquals(p90, stats["p90"]!!, TOLERANCE) + assertEquals(p99, stats["p99"]!!, TOLERANCE) + assertEquals(max, stats["max"]!!, TOLERANCE) + } +}