fix: Disable generation of criterion HTML graphs (#1728)

* fix: Disable generation of criterion HTML graphs Because we pin the benchmark runs to single cores, and criterion hence runs the report generation also on those cores, and based on `top` output it appears as this is parallelized and hence may interfere with the benchmark runs. * Try `--noplot` (and patch things so that actually works) * Missed one * Don't post comment, since this is being refactored * Fix merge --------- Signed-off-by: Lars Eggert <[email protected]>
mozilla · Mar 12, 2024 · d48fbed · d48fbed
1 parent e76645f
commit d48fbed
Show file tree

Hide file tree

Showing 8 changed files with 32 additions and 2 deletions.
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -61,7 +61,7 @@ jobs:
       - name: Run cargo bench
         run: |
           taskset -c 0 nice -n -20 \
-            cargo "+$TOOLCHAIN" bench --features bench | tee results.txt
+            cargo "+$TOOLCHAIN" bench --features bench -- --noplot | tee results.txt
 
       # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf.
       # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments.
@@ -73,7 +73,7 @@ jobs:
           mv target/criterion-transfer-profile target/criterion || true
           taskset -c 0 nice -n -20 \
             cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \
-              --bench --exact "Run multiple transfers with varying seeds"
+              --bench --exact "Run multiple transfers with varying seeds" --noplot
           # And now restore the directories.
           mv target/criterion target/criterion-transfer-profile
           mv target/criterion-bench target/criterion

diff --git a/neqo-bin/Cargo.toml b/neqo-bin/Cargo.toml
@@ -12,10 +12,12 @@ license.workspace = true
 [[bin]]
 name = "neqo-client"
 path = "src/bin/client.rs"
+bench = false
 
 [[bin]]
 name = "neqo-server"
 path = "src/bin/server/main.rs"
+bench = false
 
 [lints]
 workspace = true
@@ -35,3 +37,7 @@ qlog = { version = "0.12", default-features = false }
 regex = { version = "1.9", default-features = false, features = ["unicode-perl"] }
 tokio = { version = "1", default-features = false, features = ["net", "time", "macros", "rt", "rt-multi-thread"] }
 url = { version = "2.5", default-features = false }
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
diff --git a/neqo-common/Cargo.toml b/neqo-common/Cargo.toml
@@ -32,3 +32,7 @@ udp = ["dep:quinn-udp", "dep:tokio"]
 [target."cfg(windows)".dependencies.winapi]
 version = "0.3"
 features = ["timeapi"]
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
diff --git a/neqo-crypto/Cargo.toml b/neqo-crypto/Cargo.toml
@@ -31,3 +31,7 @@ test-fixture = { path = "../test-fixture" }
 [features]
 gecko = ["mozbuild"]
 fuzzing = []
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
diff --git a/neqo-http3/Cargo.toml b/neqo-http3/Cargo.toml
@@ -29,3 +29,7 @@ test-fixture = { path = "../test-fixture" }
 
 [features]
 fuzzing = ["neqo-transport/fuzzing", "neqo-crypto/fuzzing"]
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
diff --git a/neqo-qpack/Cargo.toml b/neqo-qpack/Cargo.toml
@@ -22,3 +22,7 @@ static_assertions = { version = "1.1", default-features = false }
 
 [dev-dependencies]
 test-fixture = { path = "../test-fixture" }
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
diff --git a/neqo-transport/Cargo.toml b/neqo-transport/Cargo.toml
@@ -29,6 +29,10 @@ test-fixture = { path = "../test-fixture" }
 bench = []
 fuzzing = ["neqo-crypto/fuzzing"]
 
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
+
 [[bench]]
 name = "transfer"
 harness = false

diff --git a/test-fixture/Cargo.toml b/test-fixture/Cargo.toml
@@ -23,3 +23,7 @@ qlog = { version = "0.12", default-features = false }
 
 [features]
 bench = []
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false