Skip to content

Bench

Bench #341

Workflow file for this run

name: Bench
on:
workflow_call:
workflow_dispatch:
schedule:
# Run at 1 AM each day, so there is a `main`-branch baseline in the cache.
- cron: '0 1 * * *'
env:
CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true
CARGO_PROFILE_RELEASE_DEBUG: true
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
TOOLCHAIN: stable
RUSTFLAGS: -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment, -C force-frame-pointers=yes
PERF_OPT: record -F997 --call-graph fp -g
permissions:
contents: read
jobs:
bench:
name: Benchmark
runs-on: self-hosted
defaults:
run:
shell: bash
steps:
- name: Checkout neqo
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
- name: Checkout msquic
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1
with:
repository: microsoft/msquic
ref: main
path: msquic
submodules: true
- name: Set PATH
run: echo "/home/bench/.cargo/bin" >> "${GITHUB_PATH}"
- name: Install Rust
uses: ./.github/actions/rust
with:
version: $TOOLCHAIN
tools: hyperfine
token: ${{ secrets.GITHUB_TOKEN }}
- name: Get minimum NSS version
id: nss-version
run: echo "minimum=$(cat neqo-crypto/min_version.txt)" >> "$GITHUB_OUTPUT"
- name: Install NSS
uses: ./.github/actions/nss
with:
minimum-version: ${{ steps.nss-version.outputs.minimum }}
- name: Build neqo
run: |
cargo "+$TOOLCHAIN" bench --workspace --features bench --no-run
cargo "+$TOOLCHAIN" build --release --bin neqo-client --bin neqo-server
- name: Build msquic
run: |
mkdir -p msquic/build
cd msquic/build
cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1 ..
cmake --build .
- name: Download cached main-branch results
id: criterion-cache
uses: actions/cache/restore@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1
with:
path: ./target/criterion
key: criterion-${{ runner.name }}-${{ github.sha }}
restore-keys: criterion-${{ runner.name }}-
# Disable turboboost, hyperthreading and use performance governor.
- name: Prepare machine
run: sudo /root/bin/prep.sh
- name: Run cargo bench
run: |
# Pin all but neqo-bin benchmarks to CPU 0. neqo-bin benchmarks run
# both a client and a server, thus benefiting from multiple CPU cores.
#
# Run all benchmarks at elevated priority.
taskset -c 0 nice -n -20 cargo "+$TOOLCHAIN" bench --workspace --exclude neqo-bin --features bench -- --noplot | tee results.txt
for MTU in 1500 65536; do
sudo ip link set dev lo mtu "$MTU"
MTU=$MTU nice -n -20 cargo "+$TOOLCHAIN" bench --package neqo-bin --features bench -- --noplot | tee -a results.txt
done
# Compare various configurations of neqo against msquic, and gather perf data
# during the hyperfine runs.
- name: Compare neqo and msquic
env:
HOST: 127.0.0.1
PORT: 4433
SIZE: 33554432 # 32 MB
run: |
TMP=$(mktemp -d)
# Make a cert and key for msquic.
openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2>/dev/null
# Make a test file for msquic to serve.
truncate -s "$SIZE" "$TMP/$SIZE"
# Define the commands to run for each client and server.
declare -A client_cmd=(
["neqo"]="target/release/neqo-client _cc _pacing --output-dir . -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
["msquic"]="msquic/build/bin/Release/quicinterop -test:D -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
)
declare -A server_cmd=(
["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT 2> /dev/null"
["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit > /dev/null || true"
)
# Replace various placeholders in the commands with the actual values.
# Also generate an extension to append to the file name.
function transmogrify {
CMD=$1
local cc=$2
local pacing=$3
if [ "$cc" != "" ]; then
CMD=${CMD//_cc/--cc $cc}
EXT="-$cc"
fi
if [ "$pacing" == "on" ]; then
CMD=${CMD//_pacing/}
EXT="$EXT-pacing"
else
CMD=${CMD//_pacing/--no-pacing}
EXT="$EXT-nopacing"
fi
}
# See https://github.com/microsoft/msquic/issues/4618#issuecomment-2422611592
for mtu in 1504 65536; do
sudo ip link set dev lo mtu "$mtu"
for server in msquic neqo; do
for client in msquic neqo; do
# msquic doesn't let us configure the congestion control or pacing.
if [ "$client" == "msquic" ] && [ "$server" == "msquic" ]; then
cc_opt=("")
pacing_opt=("")
else
cc_opt=("reno" "cubic")
pacing_opt=("on" "")
fi
for cc in "${cc_opt[@]}"; do
for pacing in "${pacing_opt[@]}"; do
# Make a tag string for this test, for the results.
TAG="$client,$server,$cc,$pacing,$mtu"
echo "Running benchmarks for $TAG" | tee -a comparison.txt
transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
# shellcheck disable=SC2086
taskset -c 0 nice -n -20 \
perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
PID=$!
transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
# shellcheck disable=SC2086
taskset -c 1 nice -n -20 \
perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
hyperfine -N --output null -w 1 -s "sleep 1" -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
tee -a comparison.txt
echo >> comparison.txt
kill $PID
cat step.md >> steps.md
# Sanity check the size of the last retrieved file.
[ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1
done
done
done
done
done
# Merge the results tables generated by hyperfine into a single table.
echo "Transfer of $SIZE bytes over loopback." > comparison.md
awk '(!/^\| Command/ || !c++) && (!/^\|:/ || !d++)' < steps.md |\
sed -E 's/`//g; s/^\|:/\|:---\|:---\|:---\|:---\|:/g; s/,/ \| /g; s/^\| Command/\| Client \| Server \| CC \| Pacing \| MTU/g' >> comparison.md
rm -r "$TMP"
# Re-enable turboboost, hyperthreading and use powersave governor.
- name: Restore machine
run: |
sudo /root/bin/unprep.sh
# In case the previous test failed:
sudo ip link set dev lo mtu 65536
if: success() || failure() || cancelled()
- name: Post-process perf data
run: |
for f in *.perf; do
# Convert for profiler.firefox.com
perf script -i "$f" -F +pid > "$f.fx" &
# Generate perf reports
perf report -i "$f" --no-children --stdio > "$f.txt" &
# Generate flamegraphs
flamegraph --perfdata "$f" --palette rust -o "${f//.perf/.svg}" &
done
wait
rm neqo.svg
- name: Format results as Markdown
id: results
run: |
{
echo "### Benchmark results"
echo
} > results.md
SHA=$(cat target/criterion/baseline-sha.txt || true)
if [ -n "$SHA" ]; then
{
echo "Performance differences relative to $SHA."
echo
} >> results.md
fi
sed -E -e 's/^ //gi' \
-e 's/((change|time|thrpt):[^%]*% )([^%]*%)(.*)/\1<b>\3<\/b>\4/gi' results.txt |\
perl -p -0777 -e 's/(.*?)\n(.*?)(((No change|Change within|Performance has).*?)(\nFound .*?)?)?\n\n/<details><summary>$1: $4<\/summary><pre>\n$2$6<\/pre><\/details>\n/gs' |\
sed -E -e 's/(Performance has regressed.)/:broken_heart: <b>\1<\/b>/gi' \
-e 's/(Performance has improved.)/:green_heart: <b>\1<\/b>/gi' \
-e 's/^ +((<\/pre>|Found).*)/\1/gi' \
-e 's/^<details>(.*Performance has.*)/<details open>\1/gi' >> results.md
{
echo
echo "### Client/server transfer results"
cat comparison.md
} >> results.md
cat results.md > "$GITHUB_STEP_SUMMARY"
- name: Remember main-branch push URL
if: github.ref == 'refs/heads/main'
run: echo "${{ github.sha }}" > target/criterion/baseline-sha.txt
- name: Store history
if: github.ref == 'refs/heads/main'
run: |
mkdir -p target/criterion-history
cp -r target/criterion "target/criterion-history/$(date +%s)-${{ github.sha }}"
- name: Cache main-branch results
if: github.ref == 'refs/heads/main'
uses: actions/cache/save@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1
with:
path: ./target/criterion
key: criterion-${{ runner.name }}-${{ github.sha }}
- name: Export perf data
id: export
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: ${{ github.event.repository.name }}-${{ github.sha }}
path: |
*.svg
*.perf
*.perf.fx
*.txt
results.*
target/criterion*
compression-level: 9
- name: Export PR comment data
uses: ./.github/actions/pr-comment-data-export
with:
name: ${{ github.workflow }}
contents: results.md
log-url: ${{ steps.export.outputs.artifact-url }}