From d48fbed733763deda5eb06099c5ae78b90f81937 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 12 Mar 2024 08:49:43 +0200
Subject: [PATCH] fix: Disable generation of criterion HTML graphs (#1728)

* fix: Disable generation of criterion HTML graphs

Because we pin the benchmark runs to single cores, and criterion hence
runs the report generation also on those cores, and based on `top`
output it appears as this is parallelized and hence may interfere
with the benchmark runs.

* Try `--noplot` (and patch things so that actually works)

* Missed one

* Don't post comment, since this is being refactored

* Fix merge

---------

Signed-off-by: Lars Eggert <lars@eggert.org>
---
 .github/workflows/bench.yml | 4 ++--
 neqo-bin/Cargo.toml         | 6 ++++++
 neqo-common/Cargo.toml      | 4 ++++
 neqo-crypto/Cargo.toml      | 4 ++++
 neqo-http3/Cargo.toml       | 4 ++++
 neqo-qpack/Cargo.toml       | 4 ++++
 neqo-transport/Cargo.toml   | 4 ++++
 test-fixture/Cargo.toml     | 4 ++++
 8 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index f19011afe3..e950671f58 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -61,7 +61,7 @@ jobs:
       - name: Run cargo bench
         run: |
           taskset -c 0 nice -n -20 \
-            cargo "+$TOOLCHAIN" bench --features bench | tee results.txt
+            cargo "+$TOOLCHAIN" bench --features bench -- --noplot | tee results.txt
 
       # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf.
       # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments.
@@ -73,7 +73,7 @@ jobs:
           mv target/criterion-transfer-profile target/criterion || true
           taskset -c 0 nice -n -20 \
             cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \
-              --bench --exact "Run multiple transfers with varying seeds"
+              --bench --exact "Run multiple transfers with varying seeds" --noplot
           # And now restore the directories.
           mv target/criterion target/criterion-transfer-profile
           mv target/criterion-bench target/criterion
diff --git a/neqo-bin/Cargo.toml b/neqo-bin/Cargo.toml
index 8b7b48ab86..ae8801eb3a 100644
--- a/neqo-bin/Cargo.toml
+++ b/neqo-bin/Cargo.toml
@@ -12,10 +12,12 @@ license.workspace = true
 [[bin]]
 name = "neqo-client"
 path = "src/bin/client.rs"
+bench = false
 
 [[bin]]
 name = "neqo-server"
 path = "src/bin/server/main.rs"
+bench = false
 
 [lints]
 workspace = true
@@ -35,3 +37,7 @@ qlog = { version = "0.12", default-features = false }
 regex = { version = "1.9", default-features = false, features = ["unicode-perl"] }
 tokio = { version = "1", default-features = false, features = ["net", "time", "macros", "rt", "rt-multi-thread"] }
 url = { version = "2.5", default-features = false }
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
diff --git a/neqo-common/Cargo.toml b/neqo-common/Cargo.toml
index 5d4d3d0d26..dae8362bfd 100644
--- a/neqo-common/Cargo.toml
+++ b/neqo-common/Cargo.toml
@@ -32,3 +32,7 @@ udp = ["dep:quinn-udp", "dep:tokio"]
 [target."cfg(windows)".dependencies.winapi]
 version = "0.3"
 features = ["timeapi"]
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
diff --git a/neqo-crypto/Cargo.toml b/neqo-crypto/Cargo.toml
index 26ec5ce067..588d084741 100644
--- a/neqo-crypto/Cargo.toml
+++ b/neqo-crypto/Cargo.toml
@@ -31,3 +31,7 @@ test-fixture = { path = "../test-fixture" }
 [features]
 gecko = ["mozbuild"]
 fuzzing = []
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
diff --git a/neqo-http3/Cargo.toml b/neqo-http3/Cargo.toml
index adb137ea15..32e3ae7e35 100644
--- a/neqo-http3/Cargo.toml
+++ b/neqo-http3/Cargo.toml
@@ -29,3 +29,7 @@ test-fixture = { path = "../test-fixture" }
 
 [features]
 fuzzing = ["neqo-transport/fuzzing", "neqo-crypto/fuzzing"]
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
diff --git a/neqo-qpack/Cargo.toml b/neqo-qpack/Cargo.toml
index 0138746cfa..c3e2ab8a66 100644
--- a/neqo-qpack/Cargo.toml
+++ b/neqo-qpack/Cargo.toml
@@ -22,3 +22,7 @@ static_assertions = { version = "1.1", default-features = false }
 
 [dev-dependencies]
 test-fixture = { path = "../test-fixture" }
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
diff --git a/neqo-transport/Cargo.toml b/neqo-transport/Cargo.toml
index 21cea4a49f..3da60bdabb 100644
--- a/neqo-transport/Cargo.toml
+++ b/neqo-transport/Cargo.toml
@@ -29,6 +29,10 @@ test-fixture = { path = "../test-fixture" }
 bench = []
 fuzzing = ["neqo-crypto/fuzzing"]
 
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false
+
 [[bench]]
 name = "transfer"
 harness = false
diff --git a/test-fixture/Cargo.toml b/test-fixture/Cargo.toml
index cc25b7d1bd..9de2a24cce 100644
--- a/test-fixture/Cargo.toml
+++ b/test-fixture/Cargo.toml
@@ -23,3 +23,7 @@ qlog = { version = "0.12", default-features = false }
 
 [features]
 bench = []
+
+[lib]
+# See https://github.com/bheisler/criterion.rs/blob/master/book/src/faq.md#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
+bench = false