diff --git a/.gitignore b/.gitignore index a2174e7..ef1ce71 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ # Dependency directories (remove the comment below to include it) # vendor/ .idea/ +*.log output/ diff --git a/scripts/benchmark_compare.sh b/scripts/benchmark_compare.sh new file mode 100755 index 0000000..329bf7e --- /dev/null +++ b/scripts/benchmark_compare.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Usage: +# `bash script/benchmark_compare.sh $old $new` +# Where old/new could be branch name, tag or commit hash. For example: +# `bash script/benchmark_compare.sh v0.7.1 develop` +# It will invoke the script/benchmark_$type.sh to run on the two versions of Kitex. +# The $type variable could be "thrift", "grpc" or "pb", as is listed in the `$types` variable below. +# +# When it finishes, the reports are saved under `output/mmdd-HHMM-$type/`. Take `output/0927-1401-thrift` +# as example, you can now run: +# `bash script/compare_result.sh output/0927-1401-thrift` +# to get the difference of two reports. + +cd `dirname $0`/.. +PROJECT_ROOT=`pwd` + +old=$1 +new=$2 + +if [ -z "$old" -o -z "$new" ]; then + echo "Usage: $0 " + echo " old/new could be branch name, tag or commit hash." + echo " e.g. $0 v1.13.1 develop" + exit 1 +fi + +types=("thrift" "grpc" "pb") + +function log_prefix() { + echo -n "[`date '+%Y-%m-%d %H:%M:%S'`] " +} + +function prepare_old() { + go get -v github.com/cloudwego/kitex@$old + go mod tidy +} + +function prepare_new() { + go get -v github.com/cloudwego/kitex@$new + go mod tidy +} + +function compare() { + type=$1 + report_dir=`date '+%m%d-%H%M'`-$type + mkdir -p $PROJECT_ROOT/output/$report_dir + log_prefix; echo "Begin comparing $type..." + + # old + log_prefix; echo "Benchmark $type @ $old (old)" + export REPORT_PREFIX=$report_dir/old- + prepare_old + time bash $PROJECT_ROOT/scripts/benchmark_$type.sh + + # new + log_prefix; echo "Benchmark $type @ $new (new)" + export REPORT_PREFIX=$report_dir/new- + prepare_new + time bash $PROJECT_ROOT/scripts/benchmark_$type.sh + + # compare results + $PROJECT_ROOT/scripts/compare_report.sh $PROJECT_ROOT/output/$report_dir + + log_prefix; echo "End comparing $type..." +} + +for tp in ${types[@]}; do + compare $tp +done + + +log_prefix; echo "All benchmark finished" diff --git a/scripts/compare_report.sh b/scripts/compare_report.sh new file mode 100755 index 0000000..6d418d7 --- /dev/null +++ b/scripts/compare_report.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# This tool is designed to be used with `scripts/benchmark_compare.sh` which generates reports with +# names like `old-xxx.csv` and `new-xxx.csv` in the `output/mmdd-HHMM-$type` directory. +# Example: +# `bash script/compare_result.sh output/0927-1401-thrift` + +cwd=`pwd` +cd `dirname $0`/.. +PROJECT_ROOT=`pwd` +cd "$cwd" + +dir=${1} +if [ -z "$dir" ];then + echo "Usage: $0 " + echo " directory can be absolute path or relative path" + exit 1 +fi + +for new in $dir/new*.csv; do + for old in $dir/old*.csv; do + echo python3 "scripts/reports/diff.py" "$old" "$new" + python3 "$PROJECT_ROOT/scripts/reports/diff.py" "$old" "$new" + done +done diff --git a/scripts/env.sh b/scripts/env.sh index 73ff222..542de4f 100755 --- a/scripts/env.sh +++ b/scripts/env.sh @@ -39,7 +39,7 @@ GOEXEC=${GOEXEC:-"go"} GOROOT=$GOROOT USER=$(whoami) -REPORT=${REPORT:-"$(date +%F-%H-%M)"} +REPORT=${REPORT_PREFIX}${REPORT:-"$(date +%F-%H-%M)"} nice_cmd='' tee_cmd="tee -a output/${REPORT}.log"