Verifying benchmark regression

.github/workflows/ci-arm64.yml

@@ -14,15 +14,35 @@ on:
         required: false
         type: string
       openssl_windows_arm64_msvc_build:
-        description: 'OpenSSL Windows Arm64 MSVC'
+        description: 'OpenSSL Windows Arm64 MSVC build'
         required: false
         default: true
         type: boolean
       openssl_windows_arm64_clangcl_build:
-        description: 'OpenSSL Windows Arm64 clang-cl'
+        description: 'OpenSSL Windows Arm64 clang-cl build'
         required: false
         default: true
         type: boolean
+      openssl_linux_aarch64_gcc_build:
+        description: 'OpenSSL Linux AArch64 gcc build'
+        required: false
+        default: true
+        type: boolean
+      benchmark:
+        description: 'Execute benchmark'
+        required: false
+        default: true
+        type: boolean
+      verify_benchmark:
+        description: 'Verify benchmark regression'
+        required: false
+        default: true
+        type: boolean
+      benchmark_snapshot:
+        description: 'Create benchmark snapshot'
+        required: false
+        default: false
+        type: boolean
   workflow_call:
     inputs:
       repository: 
@@ -36,6 +56,8 @@ on:
 
 jobs:
   build-openssl-windows-arm64-msvc:
+    if: ${{ inputs.openssl_windows_arm64_msvc_build }}
+
     runs-on: [self-hosted, Windows, ARM64, GCC, D2ps_v5]
     timeout-minutes: 600
 
@@ -61,11 +83,21 @@ jobs:
         if %errorlevel% neq 0 ( exit 1 )
         nmake test
         if %errorlevel% neq 0 ( exit 1 )
-        call .github\workflows\scripts\benchmark.bat 2> benchmark.txt
+        call .github\workflows\scripts\benchmark.bat 2> benchmark_arm64_msvc.txt
         if %errorlevel% neq 0 ( exit 1 )
-        type benchmark.txt
+        type benchmark_arm64_msvc.txt
+
+    - name: Archive openssl_windows_arm64_msvc_benchmark.zip
+      uses: actions/upload-artifact@v3
+      if: ${{ inputs.benchmark }} || ${{ inputs.benchmark_snapshot }}
+      with:
+        name: openssl_windows_arm64_msvc_benchmark.zip
+        path: benchmark_arm64_msvc.txt
+        retention-days: 3
 
   build-openssl-windows-arm64-clangcl:
+    if: ${{ inputs.openssl_windows_arm64_clangcl_build }}
+
     runs-on: [self-hosted, Windows, ARM64, GCC, D2ps_v5]
     timeout-minutes: 600
 
@@ -91,11 +123,21 @@ jobs:
         if %errorlevel% neq 0 ( exit 1 )
         nmake test
         if %errorlevel% neq 0 ( exit 1 )
-        call .github\workflows\scripts\benchmark.bat 2> benchmark.txt
+        call .github\workflows\scripts\benchmark.bat 2> benchmark_arm64_clangcl.txt
         if %errorlevel% neq 0 ( exit 1 )
-        type benchmark.txt
+        type benchmark_arm64_clangcl.txt
+
+    - name: Archive openssl_windows_arm64_clangcl_benchmark.zip
+      uses: actions/upload-artifact@v3
+      if: ${{ inputs.benchmark }} || ${{ inputs.benchmark_snapshot }}
+      with:
+        name: openssl_windows_arm64_clangcl_benchmark.zip
+        path: benchmark_arm64_clangcl.txt
+        retention-days: 3
 
   build-openssl-linux-aarch64-gcc:
+    if: ${{ inputs.openssl_linux_aarch64_gcc_build }}
+
     runs-on: [self-hosted, Linux, ARM64, GCC, D2ps_v5]
     timeout-minutes: 600
 
@@ -115,5 +157,68 @@ jobs:
         ./Configure
         make
         make test
-        .github/workflows/scripts/benchmark.sh > benchmark.txt 2>&1
-        cat benchmark.txt
+        .github/workflows/scripts/benchmark.sh > benchmark_aarch64_gcc.txt 2>&1
+        cat benchmark_aarch64_gcc.txt
+
+    - name: Archive openssl_linux_aarch64_gcc_benchmark.zip
+      uses: actions/upload-artifact@v3
+      if: ${{ inputs.benchmark }} || ${{ inputs.benchmark_snapshot }}
+      with:
+        name: openssl_linux_aarch64_gcc_benchmark.zip
+        path: benchmark_aarch64_gcc.txt
+        retention-days: 3
+
+  verify-benchmark-regression:
+    if: ${{ inputs.verify_benchmark }}
+    needs: [build-openssl-windows-arm64-clangcl, build-openssl-linux-aarch64-gcc]
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Git checkout
+      uses: actions/checkout@v3
+
+    - name: Dowload openssl_linux_aarch64_gcc_benchmark.zip
+      uses: actions/download-artifact@v3
+      with:
+        name: openssl_linux_aarch64_gcc_benchmark.zip
+
+    - name: Dowload openssl_windows_arm64_clangcl_benchmark.zip
+      uses: actions/download-artifact@v3
+      with:
+        name: openssl_windows_arm64_clangcl_benchmark.zip
+
+    - name: Verify benchmark regression
+      run: |
+        set -x
+        git fetch origin ${{ github.event.repository.default_branch }}
+        . .github/workflows/scripts/benchmark.sh verify_benchmark_regression
+
+  create-benchmark-snapshot:
+    if: ${{ inputs.benchmark_snapshot }}
+    needs: [build-openssl-windows-arm64-clangcl, build-openssl-linux-aarch64-gcc]
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Git checkout
+      uses: actions/checkout@v3
+
+    - name: Dowload openssl_linux_aarch64_gcc_benchmark.zip
+      uses: actions/download-artifact@v3
+      with:
+        name: openssl_linux_aarch64_gcc_benchmark.zip
+
+    - name: Dowload openssl_windows_arm64_clangcl_benchmark.zip
+      uses: actions/download-artifact@v3
+      with:
+        name: openssl_windows_arm64_clangcl_benchmark.zip
+
+    - name: Create PR for a benchmark snapshot
+      run: |
+        set -x
+        . .github/workflows/scripts/benchmark.sh create_benchmark_snapshot_pr
+      env:
+        BRANCH: ${{ inputs.branch }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}

.github/workflows/scripts/benchmark.sh

@@ -1,7 +1,118 @@
 #!/bin/bash
 
-LD_LIBRARY_PATH=$(pwd) apps/openssl speed
+execute_benchmark() {
+    LD_LIBRARY_PATH=$(pwd) apps/openssl speed
+
+    for i in sm3 sm4 aes-128-gcm aes-192-gcm aes-256-gcm; do
+        LD_LIBRARY_PATH="$(pwd)" apps/openssl speed -evp $i
+    done
+}
+
+calculate_Pn_position() {
+    n=$1
+    Pn=$2
+    echo "scale=0; ($n - 1) * $Pn / 100 + 1" | bc -l
+}
+
+print_Pn() {
+    Pns=$1
+    benchmark_comparison=$2
+
+    test_count=$(cat $benchmark_comparison | wc -l)
+    benchmarks=$(cat $benchmark_comparison | sort -n -k 1)
+    for i in $Pns; do 
+        echo -n "P$i "
+        echo "$benchmarks" | sed -n "$(calculate_Pn_position $test_count $i)p" | grep -o "^[^ ]*"
+    done
+}
+
+benchmark_cmp() {
+    first_benchmark=$1
+    second_benchmark=$2
+
+    cat $first_benchmark | grep "^Doing" | sed -E "s/[ \r]+$//" > b1.txt
+    cat $second_benchmark | grep "^Doing" | sed -E "s/[ \r]+$//" > b2.txt
+    paste b1.txt b2.txt | awk '{match($0, /(^[^:]+)/, g1); match($0, /\t+(Doing[^:]+)/, g2); if (g1[1] != g2[1]) exit -1}' \
+        || { echo "Benchmark test sets are not identical"; rm b1.txt b2.txt; exit -1; } 
+    cat b2.txt | sed -E "s/^[^:]+: /\/\t/" | sed -E "s/ .* / /" > b2_1.txt
+    paste b1.txt b2_1.txt | awk '{match($0, /: ([0-9]+)/, g1); match($0, /([\.0-9]+)s\t\//, g2); match($0, /\/\t([0-9]+) /, g3); match($0, /([\.0-9]+)s$/, g4); printf "%f %s\n", (g3[1] / g4[1]) / (g1[1] / g2[1]), $0}'
+    rm b1.txt b2.txt b2_1.txt
+}
+
+benchmark_snapshot() {
+    benchmark_snapshot_result=$1
+    benchmark_image=$2
+
+    benchmark_cmp benchmark_arm64_clangcl.txt benchmark_aarch64_gcc.txt > $benchmark_snapshot_result || exit -1
+
+    chart_payload="cht=bvg&chs=500x375&chtt=Perfromance%20ratio%20comparison&chma=30,30,30,30&chdlp=t&chco=4d89f9,c6d9fd&chbh=r,0,0&chxt=x,x,y&chxl=1:|Benchmark%20test%20%23&chxp=1,50&chds=a&chxs=2N*p&chdl=AArch64%20/%20Arm64%20Windows&chxr=0,0,247,48&chd=t:"
+    chart_payload+=$(cat $benchmark_snapshot_result \
+        | sort -n -r -k 1 \
+        | awk '{val = $1; if (val >= 1) val -= 1; else val = -1/val + 1; printf "%f,", val}')
+    chart_payload="${chart_payload%?}"
+
+    curl -X POST -d "$chart_payload" https://chart.googleapis.com/chart > $benchmark_image
+}
+
+get_latest_benchmark_snapshot () {
+    echo .github/benchmark_snapshot/$(ls .github/benchmark_snapshot | sort -r -k 1 | head -n 1)
+}
+
+verify_benchmark_regression() {
+    benchmark_cmp benchmark_arm64_clangcl.txt benchmark_aarch64_gcc.txt > b1 || exit -1
+    latest_benchmark_snapshot=$(get_latest_benchmark_snapshot)
+    echo The latest benchmark snapshot: $latest_benchmark_snapshot
+    print_Pn "25 50 75 95 99 100" $latest_benchmark_snapshot > Pn_latest
+    print_Pn "25 50 75 95 99 100" b1 > Pn_current
+    echo Pn latest_snapshot current percentage_change
+    paste Pn_latest Pn_current | awk 'begin{max = 0} {change = $4 * 100 / $2 - 100; if (max < change) max = change; printf "%s %s %s %f\n", $1, $2, $4, change} END {if (max > 10) print "Potential benchmark regression has been detected"}' > change
+    cat change
+    cat change | grep -q "Potential benchmark regression has been detected" && exit 1 || echo "Benchmark regression has not been detected"
+}
+
+create_benchmark_snapshot_pr() {
+    git config user.name "$GITHUB_ACTOR"
+    git config user.email "[email protected]"
+    # fetch workflow scripts from default branch
+    git fetch origin $DEFAULT_BRANCH
+    git checkout origin/$DEFAULT_BRANCH -- .github/workflows/scripts
+    branch=$BRANCH
+    # define a file for benchmark snapshot
+    [[ -z $branch ]] && branch=$(git symbolic-ref --short HEAD)
+    [ ! -d .github/benchmark_snapshot ] && mkdir -p .github/benchmark_snapshot
+    benchmark_snapshot_result=$(date +".github/benchmark_snapshot/${branch}_%Y-%m-%d_%H_%M_%S_gcc_vs_clang.txt")
+    # define a file for a benchmark image in assests branch which will be used in PR description 
+    [ ! -d .assets/benchmark ] && mkdir -p .assets/benchmark
+    benchmark_image=$(date +".assets/benchmark/benchmark_snapshot_%Y-%m-%d_%H_%M_%S_gcc_vs_clangcl.png")
+    # create benchmark snapshot and benchmark image for the PR
+    . .github/workflows/scripts/benchmark.sh benchmark_snapshot $benchmark_snapshot_result $benchmark_image
+    # add benchmark image to assets branch
+    git restore .github/workflows/scripts
+    git fetch origin assets
+    git checkout assets
+    git add $benchmark_image
+    git commit -am "* add a benchmark image"
+    git push
+    # add benchmark snapshot to PR branch
+    git checkout $DEFAULT_BRANCH --
+    git checkout -b benchmark_snapshot
+    git add .github/benchmark_snapshot
+    git commit -am "* add a benchmark snapshot"
+    git push --force origin benchmark_snapshot
+    # create a PR for benchmark snapshot
+    PR_RESPONSE=$(curl -X POST -H "Authorization: Bearer $GITHUB_TOKEN" \
+        -d '{
+        "title": "Add benchmark snapshot",
+        "body": "![image](https://raw.githubusercontent.com/Windows-on-ARM-Experiments/openssl/assets/'$benchmark_asset')",
+        "head": "benchmark_snapshot",
+        "base": "$DEFAULT_BRANCH"
+        }' \
+        https://api.github.com/repos/$GITHUB_REPOSITORY/pulls)
+    echo "$PR_RESPONSE"
+}
+
+command=$1
+shift
+
+$command "$@"
 
-for i in sm3 sm4 aes-128-gcm aes-192-gcm aes-256-gcm; do
-    LD_LIBRARY_PATH="$(pwd)" apps/openssl speed -evp $i
-done