From ba6108385244f2560475f5e4ed278612380d0259 Mon Sep 17 00:00:00 2001 From: Evgeny Karpov Date: Wed, 20 Sep 2023 00:36:33 +0200 Subject: [PATCH] Verifying benchmark regression (#6) --- .github/workflows/ci-arm64.yml | 121 +++++++++++++++++++++++-- .github/workflows/scripts/benchmark.sh | 119 +++++++++++++++++++++++- 2 files changed, 228 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-arm64.yml b/.github/workflows/ci-arm64.yml index 235882aad06cb..f661bfc09fdf0 100644 --- a/.github/workflows/ci-arm64.yml +++ b/.github/workflows/ci-arm64.yml @@ -14,15 +14,35 @@ on: required: false type: string openssl_windows_arm64_msvc_build: - description: 'OpenSSL Windows Arm64 MSVC' + description: 'OpenSSL Windows Arm64 MSVC build' required: false default: true type: boolean openssl_windows_arm64_clangcl_build: - description: 'OpenSSL Windows Arm64 clang-cl' + description: 'OpenSSL Windows Arm64 clang-cl build' required: false default: true type: boolean + openssl_linux_aarch64_gcc_build: + description: 'OpenSSL Linux AArch64 gcc build' + required: false + default: true + type: boolean + benchmark: + description: 'Execute benchmark' + required: false + default: true + type: boolean + verify_benchmark: + description: 'Verify benchmark regression' + required: false + default: true + type: boolean + benchmark_snapshot: + description: 'Create benchmark snapshot' + required: false + default: false + type: boolean workflow_call: inputs: repository: @@ -36,6 +56,8 @@ on: jobs: build-openssl-windows-arm64-msvc: + if: ${{ inputs.openssl_windows_arm64_msvc_build }} + runs-on: [self-hosted, Windows, ARM64, GCC, D2ps_v5] timeout-minutes: 600 @@ -61,11 +83,21 @@ jobs: if %errorlevel% neq 0 ( exit 1 ) nmake test if %errorlevel% neq 0 ( exit 1 ) - call .github\workflows\scripts\benchmark.bat 2> benchmark.txt + call .github\workflows\scripts\benchmark.bat 2> benchmark_arm64_msvc.txt if %errorlevel% neq 0 ( exit 1 ) - type benchmark.txt + type benchmark_arm64_msvc.txt + + - name: Archive openssl_windows_arm64_msvc_benchmark.zip + uses: actions/upload-artifact@v3 + if: ${{ inputs.benchmark }} || ${{ inputs.benchmark_snapshot }} + with: + name: openssl_windows_arm64_msvc_benchmark.zip + path: benchmark_arm64_msvc.txt + retention-days: 3 build-openssl-windows-arm64-clangcl: + if: ${{ inputs.openssl_windows_arm64_clangcl_build }} + runs-on: [self-hosted, Windows, ARM64, GCC, D2ps_v5] timeout-minutes: 600 @@ -91,11 +123,21 @@ jobs: if %errorlevel% neq 0 ( exit 1 ) nmake test if %errorlevel% neq 0 ( exit 1 ) - call .github\workflows\scripts\benchmark.bat 2> benchmark.txt + call .github\workflows\scripts\benchmark.bat 2> benchmark_arm64_clangcl.txt if %errorlevel% neq 0 ( exit 1 ) - type benchmark.txt + type benchmark_arm64_clangcl.txt + + - name: Archive openssl_windows_arm64_clangcl_benchmark.zip + uses: actions/upload-artifact@v3 + if: ${{ inputs.benchmark }} || ${{ inputs.benchmark_snapshot }} + with: + name: openssl_windows_arm64_clangcl_benchmark.zip + path: benchmark_arm64_clangcl.txt + retention-days: 3 build-openssl-linux-aarch64-gcc: + if: ${{ inputs.openssl_linux_aarch64_gcc_build }} + runs-on: [self-hosted, Linux, ARM64, GCC, D2ps_v5] timeout-minutes: 600 @@ -115,5 +157,68 @@ jobs: ./Configure make make test - .github/workflows/scripts/benchmark.sh > benchmark.txt 2>&1 - cat benchmark.txt + .github/workflows/scripts/benchmark.sh > benchmark_aarch64_gcc.txt 2>&1 + cat benchmark_aarch64_gcc.txt + + - name: Archive openssl_linux_aarch64_gcc_benchmark.zip + uses: actions/upload-artifact@v3 + if: ${{ inputs.benchmark }} || ${{ inputs.benchmark_snapshot }} + with: + name: openssl_linux_aarch64_gcc_benchmark.zip + path: benchmark_aarch64_gcc.txt + retention-days: 3 + + verify-benchmark-regression: + if: ${{ inputs.verify_benchmark }} + needs: [build-openssl-windows-arm64-clangcl, build-openssl-linux-aarch64-gcc] + + runs-on: ubuntu-latest + + steps: + - name: Git checkout + uses: actions/checkout@v3 + + - name: Dowload openssl_linux_aarch64_gcc_benchmark.zip + uses: actions/download-artifact@v3 + with: + name: openssl_linux_aarch64_gcc_benchmark.zip + + - name: Dowload openssl_windows_arm64_clangcl_benchmark.zip + uses: actions/download-artifact@v3 + with: + name: openssl_windows_arm64_clangcl_benchmark.zip + + - name: Verify benchmark regression + run: | + set -x + git fetch origin ${{ github.event.repository.default_branch }} + . .github/workflows/scripts/benchmark.sh verify_benchmark_regression + + create-benchmark-snapshot: + if: ${{ inputs.benchmark_snapshot }} + needs: [build-openssl-windows-arm64-clangcl, build-openssl-linux-aarch64-gcc] + + runs-on: ubuntu-latest + + steps: + - name: Git checkout + uses: actions/checkout@v3 + + - name: Dowload openssl_linux_aarch64_gcc_benchmark.zip + uses: actions/download-artifact@v3 + with: + name: openssl_linux_aarch64_gcc_benchmark.zip + + - name: Dowload openssl_windows_arm64_clangcl_benchmark.zip + uses: actions/download-artifact@v3 + with: + name: openssl_windows_arm64_clangcl_benchmark.zip + + - name: Create PR for a benchmark snapshot + run: | + set -x + . .github/workflows/scripts/benchmark.sh create_benchmark_snapshot_pr + env: + BRANCH: ${{ inputs.branch }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} diff --git a/.github/workflows/scripts/benchmark.sh b/.github/workflows/scripts/benchmark.sh index 3ba3cf179d2c2..a6540bb093604 100755 --- a/.github/workflows/scripts/benchmark.sh +++ b/.github/workflows/scripts/benchmark.sh @@ -1,7 +1,118 @@ #!/bin/bash -LD_LIBRARY_PATH=$(pwd) apps/openssl speed +execute_benchmark() { + LD_LIBRARY_PATH=$(pwd) apps/openssl speed + + for i in sm3 sm4 aes-128-gcm aes-192-gcm aes-256-gcm; do + LD_LIBRARY_PATH="$(pwd)" apps/openssl speed -evp $i + done +} + +calculate_Pn_position() { + n=$1 + Pn=$2 + echo "scale=0; ($n - 1) * $Pn / 100 + 1" | bc -l +} + +print_Pn() { + Pns=$1 + benchmark_comparison=$2 + + test_count=$(cat $benchmark_comparison | wc -l) + benchmarks=$(cat $benchmark_comparison | sort -n -k 1) + for i in $Pns; do + echo -n "P$i " + echo "$benchmarks" | sed -n "$(calculate_Pn_position $test_count $i)p" | grep -o "^[^ ]*" + done +} + +benchmark_cmp() { + first_benchmark=$1 + second_benchmark=$2 + + cat $first_benchmark | grep "^Doing" | sed -E "s/[ \r]+$//" > b1.txt + cat $second_benchmark | grep "^Doing" | sed -E "s/[ \r]+$//" > b2.txt + paste b1.txt b2.txt | awk '{match($0, /(^[^:]+)/, g1); match($0, /\t+(Doing[^:]+)/, g2); if (g1[1] != g2[1]) exit -1}' \ + || { echo "Benchmark test sets are not identical"; rm b1.txt b2.txt; exit -1; } + cat b2.txt | sed -E "s/^[^:]+: /\/\t/" | sed -E "s/ .* / /" > b2_1.txt + paste b1.txt b2_1.txt | awk '{match($0, /: ([0-9]+)/, g1); match($0, /([\.0-9]+)s\t\//, g2); match($0, /\/\t([0-9]+) /, g3); match($0, /([\.0-9]+)s$/, g4); printf "%f %s\n", (g3[1] / g4[1]) / (g1[1] / g2[1]), $0}' + rm b1.txt b2.txt b2_1.txt +} + +benchmark_snapshot() { + benchmark_snapshot_result=$1 + benchmark_image=$2 + + benchmark_cmp benchmark_arm64_clangcl.txt benchmark_aarch64_gcc.txt > $benchmark_snapshot_result || exit -1 + + chart_payload="cht=bvg&chs=500x375&chtt=Perfromance%20ratio%20comparison&chma=30,30,30,30&chdlp=t&chco=4d89f9,c6d9fd&chbh=r,0,0&chxt=x,x,y&chxl=1:|Benchmark%20test%20%23&chxp=1,50&chds=a&chxs=2N*p&chdl=AArch64%20/%20Arm64%20Windows&chxr=0,0,247,48&chd=t:" + chart_payload+=$(cat $benchmark_snapshot_result \ + | sort -n -r -k 1 \ + | awk '{val = $1; if (val >= 1) val -= 1; else val = -1/val + 1; printf "%f,", val}') + chart_payload="${chart_payload%?}" + + curl -X POST -d "$chart_payload" https://chart.googleapis.com/chart > $benchmark_image +} + +get_latest_benchmark_snapshot () { + echo .github/benchmark_snapshot/$(ls .github/benchmark_snapshot | sort -r -k 1 | head -n 1) +} + +verify_benchmark_regression() { + benchmark_cmp benchmark_arm64_clangcl.txt benchmark_aarch64_gcc.txt > b1 || exit -1 + latest_benchmark_snapshot=$(get_latest_benchmark_snapshot) + echo The latest benchmark snapshot: $latest_benchmark_snapshot + print_Pn "25 50 75 95 99 100" $latest_benchmark_snapshot > Pn_latest + print_Pn "25 50 75 95 99 100" b1 > Pn_current + echo Pn latest_snapshot current percentage_change + paste Pn_latest Pn_current | awk 'begin{max = 0} {change = $4 * 100 / $2 - 100; if (max < change) max = change; printf "%s %s %s %f\n", $1, $2, $4, change} END {if (max > 10) print "Potential benchmark regression has been detected"}' > change + cat change + cat change | grep -q "Potential benchmark regression has been detected" && exit 1 || echo "Benchmark regression has not been detected" +} + +create_benchmark_snapshot_pr() { + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + # fetch workflow scripts from default branch + git fetch origin $DEFAULT_BRANCH + git checkout origin/$DEFAULT_BRANCH -- .github/workflows/scripts + branch=$BRANCH + # define a file for benchmark snapshot + [[ -z $branch ]] && branch=$(git symbolic-ref --short HEAD) + [ ! -d .github/benchmark_snapshot ] && mkdir -p .github/benchmark_snapshot + benchmark_snapshot_result=$(date +".github/benchmark_snapshot/${branch}_%Y-%m-%d_%H_%M_%S_gcc_vs_clang.txt") + # define a file for a benchmark image in assests branch which will be used in PR description + [ ! -d .assets/benchmark ] && mkdir -p .assets/benchmark + benchmark_image=$(date +".assets/benchmark/benchmark_snapshot_%Y-%m-%d_%H_%M_%S_gcc_vs_clangcl.png") + # create benchmark snapshot and benchmark image for the PR + . .github/workflows/scripts/benchmark.sh benchmark_snapshot $benchmark_snapshot_result $benchmark_image + # add benchmark image to assets branch + git restore .github/workflows/scripts + git fetch origin assets + git checkout assets + git add $benchmark_image + git commit -am "* add a benchmark image" + git push + # add benchmark snapshot to PR branch + git checkout $DEFAULT_BRANCH -- + git checkout -b benchmark_snapshot + git add .github/benchmark_snapshot + git commit -am "* add a benchmark snapshot" + git push --force origin benchmark_snapshot + # create a PR for benchmark snapshot + PR_RESPONSE=$(curl -X POST -H "Authorization: Bearer $GITHUB_TOKEN" \ + -d '{ + "title": "Add benchmark snapshot", + "body": "![image](https://raw.githubusercontent.com/Windows-on-ARM-Experiments/openssl/assets/'$benchmark_asset')", + "head": "benchmark_snapshot", + "base": "$DEFAULT_BRANCH" + }' \ + https://api.github.com/repos/$GITHUB_REPOSITORY/pulls) + echo "$PR_RESPONSE" +} + +command=$1 +shift + +$command "$@" -for i in sm3 sm4 aes-128-gcm aes-192-gcm aes-256-gcm; do - LD_LIBRARY_PATH="$(pwd)" apps/openssl speed -evp $i -done