count-fail-ratio

#!/bin/bash -e
# shellcheck disable=SC2048
[ "$1" = "-h" ] || [ "$1" = "--help" ] && echo "Run an arbitrary command multiple times and count failures and fail ratio" && exit

fails="${fails:-0}"
runs="${runs:-"20"}"
start="${start:-1}"
timing="${timing:-1}"
if [ "$timing" = 1 ]; then t_start=$(date +%s%N); fi
declare -a times=()
for ((i=start; i <= runs; i++)); do
    echo "## Run $i"
    if [ "$timing" = 1 ]; then t_run_start=$(date +%s%N); fi
    $* || fails=$((fails+1))
    if [ "$timing" = 1 ]; then
        t_run_end=$(date +%s%N)
        runtime=$(( (t_run_end - t_run_start) / 1000000 ))
        times+=("$runtime")
    fi
    p=$(bc <<< "scale=9;${fails}/${i}")
    standard_error=$(bc <<< "scale=9;sqrt(${p}*(1 - ${p})/${i})")
    # critical value (z_value) for a 95% confidence level. In this
    # case, the critical value is approximately 1.96.
    z_value=1.96
    me=$(bc <<< "scale=9;${z_value}*${standard_error}")
    echo -n "## $(basename "$0"): Run: $i. Fails: $fails. Fail ratio $(bc <<< "r=${p} * 100;scale=2;r/1")±$(bc <<< "r=${me}*100;scale=2;r/1")%"
    [[ $fails = 0 ]] && echo -n ". No fails, computed failure probability < $(bc <<< "scale=2;3 * 100/${i}")%"
    echo ""
    if [ "$timing" = 1 ]; then
        t_end=$(date +%s%N)
        # Compute standard deviation
        sum=0
        for time in "${times[@]}"; do
            sum=$((sum + time))
        done
        mean=$((sum / i))
        variance=0
        for time in "${times[@]}"; do
            diff=$((time - mean))
            variance=$((variance + diff*diff))
        done
        stddev=$(bc <<< "scale=2;sqrt(${variance}/${i})")
        echo "## mean runtime: $(( (t_end - t_start) / i / 1000000 ))±$stddev ms"
    fi
done