From 6682594e631887b75894c8ab3d4e0df5f4a48047 Mon Sep 17 00:00:00 2001 From: Shaun Date: Thu, 22 Jun 2023 17:22:31 -0500 Subject: [PATCH] Introduce key/value map bench (#1121) --- opentelemetry-sdk/Cargo.toml | 9 + opentelemetry-sdk/benches/key_value_map.rs | 208 +++++++++++++++++++++ opentelemetry-sdk/benches/span_builder.rs | 107 +++++++++++ opentelemetry-sdk/benches/trace.rs | 46 +---- 4 files changed, 325 insertions(+), 45 deletions(-) create mode 100644 opentelemetry-sdk/benches/key_value_map.rs create mode 100644 opentelemetry-sdk/benches/span_builder.rs diff --git a/opentelemetry-sdk/Cargo.toml b/opentelemetry-sdk/Cargo.toml index 70e9961b5c..993afe5454 100644 --- a/opentelemetry-sdk/Cargo.toml +++ b/opentelemetry-sdk/Cargo.toml @@ -36,6 +36,7 @@ all-features = true rustdoc-args = ["--cfg", "docsrs"] [dev-dependencies] +indexmap = "1.8" criterion = { version = "0.4.0", features = ["html_reports"] } pprof = { version = "0.11.1", features = ["flamegraph", "criterion"] } @@ -50,6 +51,14 @@ rt-tokio = ["tokio", "tokio-stream"] rt-tokio-current-thread = ["tokio", "tokio-stream"] rt-async-std = ["async-std"] +[[bench]] +name = "key_value_map" +harness = false + +[[bench]] +name = "span_builder" +harness = false + [[bench]] name = "trace" harness = false diff --git a/opentelemetry-sdk/benches/key_value_map.rs b/opentelemetry-sdk/benches/key_value_map.rs new file mode 100644 index 0000000000..9a6fac5999 --- /dev/null +++ b/opentelemetry-sdk/benches/key_value_map.rs @@ -0,0 +1,208 @@ +use criterion::{ + black_box, criterion_group, criterion_main, BatchSize::SmallInput, BenchmarkId, Criterion, +}; +use indexmap::IndexMap; +use opentelemetry_api::{Key, KeyValue, Value}; +use opentelemetry_sdk::trace::EvictedHashMap; +use pprof::criterion::{Output, PProfProfiler}; +use std::iter::Iterator; + +fn criterion_benchmark(c: &mut Criterion) { + let cap = 32; + let input = [(2, 32, cap), (8, 32, cap), (32, 32, cap)]; + populate_benchmark(c, &input); + lookup_benchmark(c, &input); + populate_and_lookup_benchmark(c, &input); +} + +fn populate_benchmark(c: &mut Criterion, input: &[(usize, u32, usize)]) { + let mut group = c.benchmark_group("populate"); + for &(n, max, capacity) in input { + let parameter_string = format!("{n:02}/{max:02}/{capacity:02}"); + + group.bench_function( + BenchmarkId::new("EvictedHashMap", parameter_string.clone()), + |b| { + b.iter(|| populate_evicted_hashmap(n, max, capacity)); + }, + ); + group.bench_function( + BenchmarkId::new("IndexMap", parameter_string.clone()), + |b| { + b.iter(|| populate_indexmap(n, max, capacity)); + }, + ); + group.bench_function(BenchmarkId::new("TwoVecs", parameter_string.clone()), |b| { + b.iter(|| populate_twovecs(n, max, capacity)); + }); + group.bench_function(BenchmarkId::new("OneVec", parameter_string.clone()), |b| { + b.iter(|| populate_onevec(n, max, capacity)); + }); + } + group.finish(); +} + +fn lookup_benchmark(c: &mut Criterion, input: &[(usize, u32, usize)]) { + let mut group = c.benchmark_group("lookup"); + for &(n, max, capacity) in input { + let lookup_keys = &MAP_KEYS[n - 2..n]; + let parameter_string = format!("{n:02}/{max:02}/{capacity:02}"); + group.bench_function( + BenchmarkId::new("EvictedHashMap", parameter_string.clone()), + |b| { + b.iter_batched( + || populate_evicted_hashmap(n, max, capacity), + |map| lookup_evicted_hashmap(&map, lookup_keys), + SmallInput, + ); + }, + ); + group.bench_function( + BenchmarkId::new("IndexMap", parameter_string.clone()), + |b| { + b.iter_batched( + || populate_indexmap(n, max, capacity), + |map| lookup_indexmap(&map, lookup_keys), + SmallInput, + ); + }, + ); + group.bench_function(BenchmarkId::new("OneVec", parameter_string.clone()), |b| { + b.iter_batched( + || populate_onevec(n, max, capacity), + |vec| lookup_onevec(&vec, lookup_keys), + SmallInput, + ); + }); + group.bench_function(BenchmarkId::new("TwoVecs", parameter_string.clone()), |b| { + b.iter_batched( + || populate_twovecs(n, max, capacity), + |(keys, vals)| lookup_twovec(&keys, &vals, lookup_keys), + SmallInput, + ); + }); + } + group.finish(); +} + +fn populate_and_lookup_benchmark(c: &mut Criterion, input: &[(usize, u32, usize)]) { + let mut group = c.benchmark_group("populate_and_lookup"); + for &(n, max, capacity) in input { + let lookup_keys = &MAP_KEYS[n - 2..n]; + let parameter_string = format!("{n:02}/{max:02}/{capacity:02}"); + group.bench_function( + BenchmarkId::new("EvictedHashMap", parameter_string.clone()), + |b| { + b.iter(|| { + let map = populate_evicted_hashmap(n, max, capacity); + lookup_evicted_hashmap(&map, lookup_keys); + }); + }, + ); + group.bench_function( + BenchmarkId::new("IndexMap", parameter_string.clone()), + |b| { + b.iter(|| { + let map = populate_indexmap(n, max, capacity); + lookup_indexmap(&map, lookup_keys); + }); + }, + ); + group.bench_function(BenchmarkId::new("OneVec", parameter_string.clone()), |b| { + b.iter(|| { + let vec = populate_onevec(n, max, capacity); + lookup_onevec(&vec, lookup_keys); + }); + }); + group.bench_function(BenchmarkId::new("TwoVecs", parameter_string.clone()), |b| { + b.iter(|| { + let (keys, vals) = populate_twovecs(n, max, capacity); + lookup_twovec(&keys, &vals, lookup_keys); + }); + }); + } + group.finish(); +} + +fn populate_evicted_hashmap(n: usize, max: u32, capacity: usize) -> EvictedHashMap { + let mut map = EvictedHashMap::new(max, capacity); + for (idx, key) in MAP_KEYS.iter().enumerate().take(n) { + map.insert(KeyValue::new(*key, idx as i64)); + } + map +} + +fn lookup_evicted_hashmap(map: &EvictedHashMap, keys: &[&'static str]) { + for key in keys { + black_box(map.get(&Key::new(*key))); + } +} + +fn populate_indexmap(n: usize, max: u32, _capacity: usize) -> IndexMap { + let mut map = IndexMap::with_capacity(max as usize); + for (idx, key) in MAP_KEYS.iter().enumerate().take(n) { + map.insert(Key::new(*key), Value::I64(idx as i64)); + } + map +} + +fn lookup_indexmap(map: &IndexMap, keys: &[&'static str]) { + for key in keys { + black_box(map.get(&Key::new(*key))); + } +} + +fn populate_onevec(n: usize, max: u32, _capacity: usize) -> Vec<(Key, Value)> { + let mut tuples = Vec::with_capacity(max as usize); + for (idx, key) in MAP_KEYS.iter().enumerate().take(n) { + tuples.push((Key::new(*key), Value::I64(idx as i64))); + } + tuples +} + +fn lookup_onevec(vec: &[(Key, Value)], keys: &[&'static str]) { + for key in keys { + black_box( + vec.iter() + .position(|(k, _v)| *k == Key::new(*key)) + .map(|idx| vec.get(idx)), + ); + } +} + +fn populate_twovecs(n: usize, max: u32, _capacity: usize) -> (Vec, Vec) { + let mut keys = Vec::with_capacity(max as usize); + let mut vals = Vec::with_capacity(max as usize); + for (idx, key) in MAP_KEYS.iter().enumerate().take(n) { + keys.push(Key::new(*key)); + vals.push(Value::I64(idx as i64)); + } + (keys, vals) +} + +fn lookup_twovec(keys: &[Key], vals: &[Value], lookup_keys: &[&'static str]) { + for key in lookup_keys { + black_box( + keys.iter() + .position(|k| *k == Key::new(*key)) + .map(|idx| vals.get(idx)), + ); + } +} + +const MAP_KEYS: [&str; 64] = [ + "key.1", "key.2", "key.3", "key.4", "key.5", "key.6", "key.7", "key.8", "key.9", "key.10", + "key.11", "key.12", "key.13", "key.14", "key.15", "key.16", "key.17", "key.18", "key.19", + "key.20", "key.21", "key.22", "key.23", "key.24", "key.25", "key.26", "key.27", "key.28", + "key.29", "key.30", "key.31", "key.32", "key.33", "key.34", "key.35", "key.36", "key.37", + "key.38", "key.39", "key.40", "key.41", "key.42", "key.43", "key.44", "key.45", "key.46", + "key.47", "key.48", "key.49", "key.50", "key.51", "key.52", "key.53", "key.54", "key.55", + "key.56", "key.57", "key.58", "key.59", "key.60", "key.61", "key.62", "key.63", "key.64", +]; + +criterion_group! { + name = benches; + config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); + targets = criterion_benchmark +} +criterion_main!(benches); diff --git a/opentelemetry-sdk/benches/span_builder.rs b/opentelemetry-sdk/benches/span_builder.rs new file mode 100644 index 0000000000..cd82fe1ca6 --- /dev/null +++ b/opentelemetry-sdk/benches/span_builder.rs @@ -0,0 +1,107 @@ +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use futures_util::future::BoxFuture; +use opentelemetry_api::{ + trace::{OrderMap, Span, Tracer, TracerProvider}, + KeyValue, +}; +use opentelemetry_sdk::{ + export::trace::{ExportResult, SpanData, SpanExporter}, + trace as sdktrace, +}; +use pprof::criterion::{Output, PProfProfiler}; + +fn criterion_benchmark(c: &mut Criterion) { + span_builder_benchmark_group(c) +} + +fn span_builder_benchmark_group(c: &mut Criterion) { + let mut group = c.benchmark_group("span_builder"); + group.bench_function("simplest", |b| { + let (_provider, tracer) = not_sampled_provider(); + b.iter(|| { + let mut span = tracer.span_builder("span").start(&tracer); + span.end(); + }) + }); + group.bench_function(BenchmarkId::new("with_attributes", "1"), |b| { + let (_provider, tracer) = not_sampled_provider(); + b.iter(|| { + let mut span = tracer + .span_builder("span") + .with_attributes([KeyValue::new(MAP_KEYS[0], "value")]) + .start(&tracer); + span.end(); + }) + }); + group.bench_function(BenchmarkId::new("with_attributes", "4"), |b| { + let (_provider, tracer) = not_sampled_provider(); + b.iter(|| { + let mut span = tracer + .span_builder("span") + .with_attributes([ + KeyValue::new(MAP_KEYS[0], "value"), + KeyValue::new(MAP_KEYS[1], "value"), + KeyValue::new(MAP_KEYS[2], "value"), + KeyValue::new(MAP_KEYS[3], "value"), + ]) + .start(&tracer); + span.end(); + }) + }); + group.bench_function(BenchmarkId::new("with_attributes_map", "1"), |b| { + let (_provider, tracer) = not_sampled_provider(); + b.iter(|| { + let mut span = tracer + .span_builder("span") + .with_attributes_map(OrderMap::from_iter([KeyValue::new(MAP_KEYS[0], "value")])) + .start(&tracer); + span.end(); + }) + }); + group.bench_function(BenchmarkId::new("with_attributes_map", "4"), |b| { + let (_provider, tracer) = not_sampled_provider(); + b.iter(|| { + let mut span = tracer + .span_builder("span") + .with_attributes_map(OrderMap::from_iter([KeyValue::new(MAP_KEYS[0], "value")])) + .start(&tracer); + span.end(); + }) + }); + group.finish(); +} + +fn not_sampled_provider() -> (sdktrace::TracerProvider, sdktrace::Tracer) { + let provider = sdktrace::TracerProvider::builder() + .with_config(sdktrace::config().with_sampler(sdktrace::Sampler::AlwaysOff)) + .with_simple_exporter(NoopExporter) + .build(); + let tracer = provider.tracer("not-sampled"); + (provider, tracer) +} + +#[derive(Debug)] +struct NoopExporter; + +impl SpanExporter for NoopExporter { + fn export(&mut self, _spans: Vec) -> BoxFuture<'static, ExportResult> { + Box::pin(futures_util::future::ready(Ok(()))) + } +} + +const MAP_KEYS: [&str; 64] = [ + "key.1", "key.2", "key.3", "key.4", "key.5", "key.6", "key.7", "key.8", "key.9", "key.10", + "key.11", "key.12", "key.13", "key.14", "key.15", "key.16", "key.17", "key.18", "key.19", + "key.20", "key.21", "key.22", "key.23", "key.24", "key.25", "key.26", "key.27", "key.28", + "key.29", "key.30", "key.31", "key.32", "key.33", "key.34", "key.35", "key.36", "key.37", + "key.38", "key.39", "key.40", "key.41", "key.42", "key.43", "key.44", "key.45", "key.46", + "key.47", "key.48", "key.49", "key.50", "key.51", "key.52", "key.53", "key.54", "key.55", + "key.56", "key.57", "key.58", "key.59", "key.60", "key.61", "key.62", "key.63", "key.64", +]; + +criterion_group! { + name = benches; + config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); + targets = criterion_benchmark +} +criterion_main!(benches); diff --git a/opentelemetry-sdk/benches/trace.rs b/opentelemetry-sdk/benches/trace.rs index 82e1006656..d45ed69dfa 100644 --- a/opentelemetry-sdk/benches/trace.rs +++ b/opentelemetry-sdk/benches/trace.rs @@ -2,7 +2,7 @@ use criterion::{criterion_group, criterion_main, Criterion}; use futures_util::future::BoxFuture; use opentelemetry_api::{ trace::{Span, Tracer, TracerProvider}, - Key, KeyValue, + Key, }; use opentelemetry_sdk::{ export::trace::{ExportResult, SpanData, SpanExporter}, @@ -11,21 +11,6 @@ use opentelemetry_sdk::{ use pprof::criterion::{Output, PProfProfiler}; fn criterion_benchmark(c: &mut Criterion) { - let mut group = c.benchmark_group("EvictedHashMap"); - group.bench_function("insert 1", |b| { - b.iter(|| insert_keys(sdktrace::EvictedHashMap::new(32, 1), 1)) - }); - group.bench_function("insert 5", |b| { - b.iter(|| insert_keys(sdktrace::EvictedHashMap::new(32, 5), 5)) - }); - group.bench_function("insert 10", |b| { - b.iter(|| insert_keys(sdktrace::EvictedHashMap::new(32, 10), 10)) - }); - group.bench_function("insert 20", |b| { - b.iter(|| insert_keys(sdktrace::EvictedHashMap::new(32, 20), 20)) - }); - group.finish(); - trace_benchmark_group(c, "start-end-span", |tracer| tracer.start("foo").end()); trace_benchmark_group(c, "start-end-span-4-attrs", |tracer| { @@ -70,35 +55,6 @@ fn criterion_benchmark(c: &mut Criterion) { }); } -const MAP_KEYS: [Key; 20] = [ - Key::from_static_str("key1"), - Key::from_static_str("key2"), - Key::from_static_str("key3"), - Key::from_static_str("key4"), - Key::from_static_str("key5"), - Key::from_static_str("key6"), - Key::from_static_str("key7"), - Key::from_static_str("key8"), - Key::from_static_str("key9"), - Key::from_static_str("key10"), - Key::from_static_str("key11"), - Key::from_static_str("key12"), - Key::from_static_str("key13"), - Key::from_static_str("key14"), - Key::from_static_str("key15"), - Key::from_static_str("key16"), - Key::from_static_str("key17"), - Key::from_static_str("key18"), - Key::from_static_str("key19"), - Key::from_static_str("key20"), -]; - -fn insert_keys(mut map: sdktrace::EvictedHashMap, n: usize) { - for (idx, key) in MAP_KEYS.iter().enumerate().take(n) { - map.insert(KeyValue::new(key.clone(), idx as i64)); - } -} - #[derive(Debug)] struct VoidExporter;