Skip to content

Commit

Permalink
introduce BenchGroup
Browse files Browse the repository at this point in the history
  • Loading branch information
PSeitz committed May 19, 2024
1 parent c3190ac commit f1b529d
Show file tree
Hide file tree
Showing 9 changed files with 276 additions and 139 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ name = "fibonacci_bench"
harness = false

[[bench]]
name = "bench"
name = "bench_group"
harness = false

[[bench]]
Expand Down
9 changes: 5 additions & 4 deletions benches/bench.rs → benches/bench_group.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,17 @@ fn run_bench() {
runner.enable_perf();

runner.set_cache_trasher(true);
let mut group = runner.new_group();
for (input_name, data) in inputs.iter() {
runner.set_input_size(data.len() * std::mem::size_of::<usize>());
runner.register_with_input("vec", input_name, data, move |data| {
group.set_input_size(data.len() * std::mem::size_of::<usize>());
group.register_with_input("vec", input_name, data, move |data| {
black_box(test_vec(data));
});
runner.register_with_input("hashmap", input_name, data, move |data| {
group.register_with_input("hashmap", input_name, data, move |data| {
black_box(test_hashmap(data));
});
}
runner.run();
group.run();
}

fn main() {
Expand Down
6 changes: 3 additions & 3 deletions src/bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ pub trait Bench<'a> {
/// Sample the number of iterations the benchmark should do
fn sample_num_iter(&self) -> usize;
fn exec_bench(&mut self, alloc: &Option<Alloc>);
fn get_results(&mut self, group_name: &Option<String>) -> BenchResult;
fn get_results(&mut self, group_name: Option<&str>) -> BenchResult;
fn clear_results(&mut self);
}

Expand Down Expand Up @@ -99,10 +99,10 @@ impl<'a, I> Bench<'a> for InputWithBenchmark<'a, I> {
self.results.push(res);
}

fn get_results(&mut self, group_name: &Option<String>) -> BenchResult {
fn get_results(&mut self, group_name: Option<&str>) -> BenchResult {
let bench_id = format!(
"{}_{}_{}",
group_name.as_ref().unwrap_or(&"".to_string()),
group_name.as_ref().unwrap_or(&""),
self.input.name,
self.bench.name
)
Expand Down
129 changes: 129 additions & 0 deletions src/bench_group.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
use std::borrow::Cow;

use crate::{
bench::{Bench, InputWithBenchmark, NamedBench},
bench_runner::{BenchRunner, EMPTY_INPUT},
NamedInput,
};

/// `BenchGroup` is a group of benchmarks run together.
///
pub struct BenchGroup<'a> {
name: Option<String>,
pub(crate) benches: Vec<Box<dyn Bench<'a> + 'a>>,
/// The size of the input.
/// Enables throughput reporting.
input_size_in_bytes: Option<usize>,
pub(crate) runner: BenchRunner,
}

impl<'a> BenchGroup<'a> {
/// Create a new BenchGroup with no benchmarks.
pub fn new(runner: BenchRunner) -> Self {
Self {
name: None,
benches: Vec::new(),
input_size_in_bytes: None,
runner,
}
}

/// Create a new BenchGroup with no benchmarks.
pub fn with_name<S: Into<String>>(runner: BenchRunner, name: S) -> Self {
Self {
name: Some(name.into()),
benches: Vec::new(),
input_size_in_bytes: None,
runner,
}
}

/// Sets name of the group and returns the group.
pub fn name<S: Into<String>>(mut self, name: S) -> Self {
self.name = Some(name.into());
self
}

/// Enables throughput reporting. The throughput will be valid for all inputs that are
/// registered afterwards.
pub fn set_input_size(&mut self, input_size: usize) {
self.input_size_in_bytes = Some(input_size);
}

/// Register a benchmark with the given name and function.
pub fn register_with_input<I, F, S: Into<String>>(
&mut self,
bench_name: S,
input_name: S,
input: &'a I,
fun: F,
) where
F: Fn(&'a I) + 'static,
{
let name = bench_name.into();
let input_name = input_name.into();

let bench = NamedBench::new(name, Box::new(fun));
self.register_named_with_input(
bench,
NamedInput {
name: Cow::Owned(input_name),
data: input,
},
);
}

/// Register a benchmark with the given name and function.
pub fn register<I, F, S: Into<String>>(&mut self, name: S, fun: F)
where
F: Fn(&'a ()) + 'static,
{
let name = name.into();
let bench = NamedBench::new(name, Box::new(fun));

self.register_named_with_input(bench, EMPTY_INPUT);
}

/// Register a benchmark with the given name and function.
pub(crate) fn register_named_with_input<I>(
&mut self,
bench: NamedBench<'a, I>,
input: NamedInput<'a, I>,
) {
if let Some(filter) = &self.runner.options.filter {
if !bench.name.contains(filter) && !input.name.contains(filter) {
return;
}
}

let bundle = InputWithBenchmark::new(
input,
self.input_size_in_bytes,
bench,
self.runner.options.enable_perf,
);

self.benches.push(Box::new(bundle));
}

/// Set the name of the group.
/// The name is printed before the benchmarks are run.
/// It is also used to distinguish when writing the results to disk.
pub fn set_name<S: Into<String>>(&mut self, name: S) {
self.name = Some(name.into());
}

/// Sets the filter, which is used to filter the benchmarks by name.
/// The filter is fetched from the command line arguments.
///
/// It can also match an input name.
pub fn set_filter(&mut self, filter: Option<String>) {
self.runner.set_filter(filter);
}

/// Run the benchmarks and report the results.
pub fn run(&mut self) {
self.runner
.run_group(self.name.as_deref(), &mut self.benches);
}
}
57 changes: 40 additions & 17 deletions src/bench_input_group.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
use std::{alloc::GlobalAlloc, borrow::Cow, collections::HashMap};

use crate::{bench::NamedBench, bench_runner::BenchRunner, parse_args, NamedInput, Options};
use crate::{
bench::NamedBench,
bench_runner::{group_by_mut, BenchRunner},
parse_args, BenchGroup, NamedInput, Options,
};
use peakmem_alloc::*;
use yansi::Paint;

pub(crate) type Alloc = &'static dyn PeakMemAllocTrait;

Expand All @@ -13,7 +18,8 @@ pub(crate) type Alloc = &'static dyn PeakMemAllocTrait;
/// to the `InputGroup`. If this is not possible, use [BenchRunner](crate::BenchRunner) instead.
pub struct InputGroup<I = ()> {
inputs: Vec<OwnedNamedInput<I>>,
runner: BenchRunner<'static>,
bench_group: BenchGroup<'static>,
pub(crate) name: Option<String>,
}

impl Default for InputGroup<()> {
Expand All @@ -39,7 +45,7 @@ pub struct OwnedNamedInput<I> {
impl<I: 'static> InputGroup<I> {
/// Sets name of the group and returns the group.
pub fn name<S: Into<String>>(mut self, name: S) -> Self {
self.runner.set_name(name.into());
self.name = Some(name.into());
self
}
/// The inputs are a vector of tuples, where the first element is the name of the input and the
Expand Down Expand Up @@ -67,12 +73,16 @@ impl<I: 'static> InputGroup<I> {
let mut runner = BenchRunner::new();
runner.set_options(options);

InputGroup { inputs, runner }
InputGroup {
inputs,
name: None,
bench_group: BenchGroup::new(runner),
}
}
/// Set the peak mem allocator to be used for the benchmarks.
/// This will report the peak memory consumption of the benchmarks.
pub fn set_alloc<A: GlobalAlloc + 'static>(&mut self, alloc: &'static PeakMemAlloc<A>) {
self.runner.set_alloc(alloc);
self.bench_group.runner.set_alloc(alloc);
}
/// Enable perf profiling + report
///
Expand All @@ -91,7 +101,7 @@ impl<I: 'static> InputGroup<I> {
/// L1dA: 2.001 L1dM: 0.000 Br: 6.001 BrM: 0.000
/// ```
pub fn enable_perf(&mut self) {
self.runner.options.enable_perf = true;
self.bench_group.runner.options.enable_perf = true;
}

/// Enables throughput reporting.
Expand All @@ -108,41 +118,41 @@ impl<I: 'static> InputGroup<I> {
/// Set the name of the group.
/// The name is printed before the benchmarks are run.
/// It is also used to distinguish when writing the results to disk.
pub fn set_name(&mut self, name: String) {
self.runner.set_name(name);
pub fn set_name<S: Into<String>>(&mut self, name: S) {
self.name = Some(name.into());
}

/// Set the options to the given value.
/// This will overwrite all current options.
///
/// See the Options struct for more information.
pub fn set_options(&mut self, options: Options) {
self.runner.set_options(options);
self.bench_group.runner.set_options(options);
}

/// Manully set the number of iterations each benchmark is called.
///
/// This disables the automatic detection of the number of iterations.
pub fn set_num_iter(&mut self, num_iter: usize) {
self.runner.set_num_iter(num_iter);
self.bench_group.runner.set_num_iter(num_iter);
}

/// Trash CPU cache between bench runs. Defaults to false.
pub fn set_cache_trasher(&mut self, enable: bool) {
self.runner.set_cache_trasher(enable);
self.bench_group.runner.set_cache_trasher(enable);
}

/// Sets the interleave option to the given value.
pub fn set_interleave(&mut self, interleave: bool) {
self.runner.set_interleave(interleave);
self.bench_group.runner.set_interleave(interleave);
}

/// Sets the filter, which is used to filter the benchmarks by name.
/// The filter is fetched from the command line arguments.
///
/// It can also match an input name.
pub fn set_filter(&mut self, filter: Option<String>) {
self.runner.set_filter(filter);
self.bench_group.runner.set_filter(filter);
}

/// Register a benchmark with the given name and function.
Expand All @@ -163,25 +173,38 @@ impl<I: 'static> InputGroup<I> {
// (probably).
let named_input: NamedInput<'static, I> = unsafe { transmute_lifetime(named_input) };
if let Some(input_size) = input.input_size_in_bytes {
self.runner.set_input_size(input_size);
self.bench_group.runner.set_input_size(input_size);
}
self.runner
self.bench_group
.register_named_with_input(named_bench, named_input);
}
}

/// Run the benchmarks and report the results.
pub fn run(&mut self) {
if let Some(name) = &self.name {
println!("{}", name.black().on_red().invert().bold());
}
let input_name_to_ordinal: HashMap<String, usize> = self
.inputs
.iter()
.enumerate()
.map(|(i, input)| (input.name.clone(), i))
.collect();
self.runner
self.bench_group
.benches
.sort_by_key(|bench| std::cmp::Reverse(input_name_to_ordinal[bench.get_input_name()]));
self.runner.run();
group_by_mut(
self.bench_group.benches.as_mut_slice(),
|b| b.get_input_name(),
|group| {
let input_name = group[0].get_input_name().to_owned();
//if !input_name.is_empty() {
//println!("{}", input_name.black().on_yellow().invert().italic());
//}
self.bench_group.runner.run_group(Some(&input_name), group);
},
);
}
}

Expand Down
Loading

0 comments on commit f1b529d

Please sign in to comment.