From d6094c64fab22cc95fbfc45dd0e5a781663cd28e Mon Sep 17 00:00:00 2001 From: Raphael Druon Date: Fri, 2 Aug 2024 01:27:51 -0600 Subject: [PATCH] Add criterion benchmark --- .github/workflows/bench.yml | 12 ++ Cargo.lock | 226 ++++++++++++++++++++++ lustre-collector/Cargo.toml | 9 + lustre-collector/benches/jobstats_fast.rs | 49 +++++ lustre-collector/benches/jobstats_slow.rs | 52 +++++ 5 files changed, 348 insertions(+) create mode 100644 .github/workflows/bench.yml create mode 100644 lustre-collector/benches/jobstats_fast.rs create mode 100644 lustre-collector/benches/jobstats_slow.rs diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..4f00601 --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,12 @@ +name: benchmark pull requests +on: [pull_request] +jobs: + runBenchmark: + name: run benchmark + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: boa-dev/criterion-compare-action@v3 + with: + cwd: "lustre-collector" + branchName: ${{ github.base_ref }} diff --git a/Cargo.lock b/Cargo.lock index f103c23..3a62557 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "0.6.14" @@ -137,6 +143,12 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.97" @@ -149,6 +161,33 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.5.4" @@ -234,6 +273,79 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "encode_unicode" version = "0.3.6" @@ -331,6 +443,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -478,12 +600,32 @@ dependencies = [ "similar", ] +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -545,6 +687,7 @@ version = "0.8.2" dependencies = [ "clap", "combine", + "criterion", "include_dir", "insta", "serde", @@ -716,6 +859,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "oorandom" +version = "11.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" + [[package]] name = "openssl-probe" version = "0.1.5" @@ -763,6 +912,34 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plotters" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" + +[[package]] +name = "plotters-svg" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +dependencies = [ + "plotters-backend", +] + [[package]] name = "proc-macro2" version = "1.0.82" @@ -820,6 +997,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.1" @@ -961,6 +1158,15 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.23" @@ -1171,6 +1377,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" version = "1.37.0" @@ -1320,6 +1536,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" diff --git a/lustre-collector/Cargo.toml b/lustre-collector/Cargo.toml index 0c228f8..c49d578 100644 --- a/lustre-collector/Cargo.toml +++ b/lustre-collector/Cargo.toml @@ -19,3 +19,12 @@ tracing.workspace = true [dev-dependencies] include_dir.workspace = true insta.workspace = true +criterion = { version = "0.5", features = ["html_reports"] } + +[[bench]] +name = "jobstats_slow" +harness = false + +[[bench]] +name = "jobstats_fast" +harness = false diff --git a/lustre-collector/benches/jobstats_fast.rs b/lustre-collector/benches/jobstats_fast.rs new file mode 100644 index 0000000..2465fa8 --- /dev/null +++ b/lustre-collector/benches/jobstats_fast.rs @@ -0,0 +1,49 @@ +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use lustre_collector::{parse_lctl_output, Record}; + +fn test_data(repeat: usize) -> String { + let job = r#" + - job_id: "SLURM_JOB_machine184_74186:0:ma" + snapshot_time: 1720516680 + read_bytes: { samples: 0, unit: bytes, min: 0, max: 0, sum: 0, sumsq: 0 } + write_bytes: { samples: 52, unit: bytes, min: 4096, max: 475136, sum: 5468160, sumsq: 1071040692224 } + read: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + write: { samples: 52, unit: usecs, min: 12, max: 40081, sum: 692342, sumsq: 17432258604 } + getattr: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + setattr: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + punch: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + sync: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + destroy: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + create: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + statfs: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + get_info: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + set_info: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + quotactl: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + prealloc: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 }"#; + let lctl_output = r#"obdfilter.ds002-OST0000.job_stats= +job_stats:"#; + + let input = format!("{lctl_output}{}", job.to_string().repeat(repeat)); + + input +} + +fn parse_jobstats(repeat: usize) -> Vec { + let input = test_data(repeat); + parse_lctl_output(input.as_bytes()).unwrap() +} + +fn criterion_benchmark_fast(c: &mut Criterion) { + c.bench_function("jobstats 100", |b| { + b.iter(|| parse_jobstats(black_box(100))) + }); + c.bench_function("jobstats 1000", |b| { + b.iter(|| parse_jobstats(black_box(1000))) + }); +} +criterion_group! { + name = benches; + config = Criterion::default(); + targets = criterion_benchmark_fast +} +criterion_main!(benches); diff --git a/lustre-collector/benches/jobstats_slow.rs b/lustre-collector/benches/jobstats_slow.rs new file mode 100644 index 0000000..d7ec1e0 --- /dev/null +++ b/lustre-collector/benches/jobstats_slow.rs @@ -0,0 +1,52 @@ +use std::time::Duration; + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use lustre_collector::{parse_lctl_output, Record}; + +fn test_data(repeat: usize) -> String { + let job = r#" + - job_id: "SLURM_JOB_machine184_74186:0:ma" + snapshot_time: 1720516680 + read_bytes: { samples: 0, unit: bytes, min: 0, max: 0, sum: 0, sumsq: 0 } + write_bytes: { samples: 52, unit: bytes, min: 4096, max: 475136, sum: 5468160, sumsq: 1071040692224 } + read: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + write: { samples: 52, unit: usecs, min: 12, max: 40081, sum: 692342, sumsq: 17432258604 } + getattr: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + setattr: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + punch: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + sync: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + destroy: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + create: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + statfs: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + get_info: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + set_info: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + quotactl: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 } + prealloc: { samples: 0, unit: usecs, min: 0, max: 0, sum: 0, sumsq: 0 }"#; + let lctl_output = r#"obdfilter.ds002-OST0000.job_stats= +job_stats:"#; + + let input = format!("{lctl_output}{}", job.to_string().repeat(repeat)); + + input +} + +fn parse_jobstats(repeat: usize) -> Vec { + let input = test_data(repeat); + parse_lctl_output(input.as_bytes()).unwrap() +} + +fn criterion_benchmark_slow(c: &mut Criterion) { + c.bench_function("jobstats 10000", |b| { + b.iter(|| parse_jobstats(black_box(10000))) + }); + c.bench_function("jobstats 100000", |b| { + b.iter(|| parse_jobstats(black_box(100000))) + }); +} + +criterion_group! { + name = benches; + config = Criterion::default().sample_size(10).measurement_time(Duration::from_secs(600)); + targets = criterion_benchmark_slow +} +criterion_main!(benches);