Skip to content

Commit

Permalink
Run bash and dash tests together
Browse files Browse the repository at this point in the history 
Also changes it so that only the bash tests only
run in bash mode, which I feel is fair since they
test bash only features
  • Loading branch information
@BolunThompson
BolunThompson committed Dec 26, 2024
1 parent 441020a commit 3c2606b
Show file tree
Hide file tree
Showing 6 changed files with 143 additions and 166 deletions.
32 changes: 18 additions & 14 deletions evaluation/intro/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,9 @@

export PASH_TOP=${PASH_TOP:-$(git rev-parse --show-toplevel --show-superproject-working-tree)}

# are we running in bash mode?
test_mode=${1:-dash}
if [ "$test_mode" = "bash" ]; then
echo "Running in bash mode"
fi

## TODO: Make the compiler work too.
bash="bash"
if [ "$test_mode" = "bash" ]; then
echo "Running in bash mode confirmed"
pash="$PASH_TOP/pa.sh -w 2 --bash"
else
pash="$PASH_TOP/pa.sh -w 2"
fi
pash="$PASH_TOP/pa.sh -w 2"
pash_with_bash="$PASH_TOP/pa.sh -w 2 --bash"

output_dir="$PASH_TOP/evaluation/intro/output"
rm -rf "$output_dir"
Expand Down Expand Up @@ -45,6 +34,22 @@ run_test()
echo "$test are not identical" >> $output_dir/result_status
echo -e '\t\tFAIL'
return 1
fi
TIMEFORMAT="%3R"
{ time $pash_with_bash "$test" > "$output_dir/$test.pash_bash.out"; } 2> >(tee -a $output_dir/results.time_pash_bash)
test_pash_bash_ec=$?
diff "$output_dir/$test.bash.out" "$output_dir/$test.pash_bash.out"
test_diff_bash_ec=$?
if [ $test_diff_bash_ec -ne 0 ]; then
echo -n "$test output mismatch"
fi
if [ $test_bash_ec -ne $test_pash_bash_ec ]; then
echo -n "$test exit code mismatch"
fi
if [ $test_diff_bash_ec -ne 0 ] || [ $test_bash_ec -ne $test_pash_bash_ec ]; then
echo "$test are not identical" >> $output_dir/result_status
echo -e '\t\tFAIL'
return 1
else
echo "$test are identical" >> $output_dir/result_status
echo -e '\t\tOK'
Expand All @@ -62,7 +67,6 @@ elif [ -e /etc/os-release ] ; then
fi

distro=$(printf '%s\n' "$distro" | LC_ALL=C tr '[:upper:]' '[:lower:]')
# now do different things depending on distro
case "$distro" in
freebsd*)
# change sed to gsed
Expand Down
144 changes: 73 additions & 71 deletions evaluation/tests/interface_tests/run.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,8 @@
export PASH_TOP=${PASH_TOP:-$(git rev-parse --show-toplevel --show-superproject-working-tree)}
# time: print real in seconds, to simplify parsing

# are we running in bash mode?
test_mode=${1:-dash}
if [ "$test_mode" = "bash" ]; then
echo "Running in bash mode"
fi

bash="bash"
if [ "$test_mode" = "bash" ]; then
echo "Running in bash mode confirmed"
pash="$PASH_TOP/pa.sh --parallel_pipelines --bash"
else
pash="$PASH_TOP/pa.sh --parallel_pipelines"
fi
pash="$PASH_TOP/pa.sh --parallel_pipelines"
pash_with_bash="$PASH_TOP/pa.sh --parallel_pipelines --bash"

output_dir="$PASH_TOP/evaluation/tests/interface_tests/output"
rm -rf "$output_dir"
Expand Down Expand Up @@ -50,11 +39,32 @@ run_test()
echo -n "$test exit code mismatch "
fi
if [ $test_diff_ec -ne 0 ] || [ $test_ec -ne 0 ]; then
echo "$test are not identical" >> $output_dir/result_status
echo "$test are not identical (pash)" >> "$output_dir/result_status"
echo -e '\t\tFAIL'
return 1
fi

{ time $test "$pash_with_bash" > "$output_dir/$test.pash_bash.out"; } 2>> "$output_dir/results.time_pash_bash"
test_pash_bash_ec=$?
diff "$output_dir/$test.bash.out" "$output_dir/$test.pash_bash.out"
test_diff_bash_ec=$?

# Check if exit codes both succeed or both fail
{ [ $test_bash_ec -eq 0 ] && [ $test_pash_bash_ec -eq 0 ]; } || { [ $test_bash_ec -ne 0 ] && [ $test_pash_bash_ec -ne 0 ]; }
test_ec_bash=$?

if [ $test_diff_bash_ec -ne 0 ]; then
echo -n "$test output mismatch "
fi
if [ $test_ec_bash -ne 0 ]; then
echo -n "$test exit code mismatch "
fi
if [ $test_diff_bash_ec -ne 0 ] || [ $test_ec_bash -ne 0 ]; then
echo "$test are not identical (pash --bash)" >> "$output_dir/result_status"
echo -e '\t\tFAIL'
return 1
else
echo "$test are identical" >> $output_dir/result_status
echo "$test are identical" >> "$output_dir/result_status"
echo -e '\t\tOK'
return 0
fi
Expand Down Expand Up @@ -338,57 +348,49 @@ test_IFS()
$shell test-IFS.sh
}

## We run all tests composed with && to exit on the first that fails
if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
run_test test1
run_test test2
run_test test3
run_test test4
run_test test5
run_test test6
# run_test test7
run_test test8
run_test test9
run_test test10
run_test test11
run_test test12
run_test test13
run_test test14
run_test test15
run_test test16
run_test test17
run_test test18
run_test test_set
run_test test_set_e
run_test test_redirect
run_test test_unparsing
run_test test_set_e_2
run_test test_set_e_3
run_test test_new_line_in_var
run_test test_cmd_sbst
run_test test_cmd_sbst2
run_test test_exec_redirections
run_test test_cat_hyphen
run_test test_trap
run_test test_set-dash
run_test test_cat_redir_fail
run_test test_umask
run_test test_expand_u
run_test test_expand_u_positional
run_test test_quoting
run_test test_var_assgn_default
run_test test_exclam
run_test test_redir_var_test
run_test test_star
run_test test_env_vars
run_test test_redir_dup
run_test test_IFS
else
for testname in $@
do
run_test "$testname"
done
fi
run_test test1
run_test test2
run_test test3
run_test test4
run_test test5
run_test test6
# run_test test7
run_test test8
run_test test9
run_test test10
run_test test11
run_test test12
run_test test13
run_test test14
run_test test15
run_test test16
run_test test17
run_test test18
run_test test_set
run_test test_set_e
run_test test_redirect
run_test test_unparsing
run_test test_set_e_2
run_test test_set_e_3
run_test test_new_line_in_var
run_test test_cmd_sbst
run_test test_cmd_sbst2
run_test test_exec_redirections
run_test test_cat_hyphen
run_test test_trap
run_test test_set-dash
run_test test_cat_redir_fail
run_test test_umask
run_test test_expand_u
run_test test_expand_u_positional
run_test test_quoting
run_test test_var_assgn_default
run_test test_exclam
run_test test_redir_var_test
run_test test_star
run_test test_env_vars
run_test test_redir_dup
run_test test_IFS

if type lsb_release >/dev/null 2>&1 ; then
distro=$(lsb_release -i -s)
Expand All @@ -402,21 +404,21 @@ case "$distro" in
freebsd*)
# change sed to gsed
sed () {
gsed $@
gsed "$@"
}
;;
*)
;;
esac

echo "group,Bash,Pash-DRY_COMP" > $output_dir/results.time
paste $output_dir/results.time_* | sed 's\,\.\g' | sed 's\:\,\g' | sed 's/\t/,/' >> $output_dir/results.time
echo "group,Bash,Pash-DRY_COMP" > "$output_dir/results.time"
paste "$output_dir"/results.time_* | sed 's\,\.\g' | sed 's\:\,\g' | sed 's/\t/,/' >> "$output_dir/results.time"

echo "Below follow the identical outputs:"
grep "are identical" "$output_dir"/result_status | awk '{print $1}'
grep "are identical" "$output_dir"/result_status | awk '{print $1, $5}'

echo "Below follow the non-identical outputs:"
grep "are not identical" "$output_dir"/result_status | awk '{print $1}'
grep "are not identical" "$output_dir"/result_status | awk '{print $1, $5}'

TOTAL_TESTS=$(cat "$output_dir"/result_status | wc -l)
PASSED_TESTS=$(grep -c "are identical" "$output_dir"/result_status)
Expand Down
41 changes: 13 additions & 28 deletions evaluation/tests/interface_tests/run_bash_tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,8 @@
export PASH_TOP=${PASH_TOP:-$(git rev-parse --show-toplevel --show-superproject-working-tree)}
# time: print real in seconds, to simplify parsing

# are we running in bash mode?
test_mode=${1:-dash}
if [ "$test_mode" = "bash" ]; then
echo "Running in bash mode"
fi

bash="bash"
if [ "$test_mode" = "bash" ]; then
echo "Running in bash mode confirmed"
pash="$PASH_TOP/pa.sh --parallel_pipelines --profile_driven --bash"
else
pash="$PASH_TOP/pa.sh --parallel_pipelines --profile_driven"
fi
pash="$PASH_TOP/pa.sh --parallel_pipelines --profile_driven --bash"

output_dir="$PASH_TOP/evaluation/tests/interface_tests/output"
rm -rf "$output_dir"
Expand Down Expand Up @@ -2900,12 +2889,8 @@ test_select.sh()
}

## We run all tests composed with && to exit on the first that fails
# commented out IFS tests should work but this is an issue with PaSh, bug report made
if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
cd bash_tests
if [ "$#" -eq 0 ]; then
echo "Warning: these tests should be run with bash as the first argument to test the bash mode."
fi
# run_test test_exec8.sub - any uncommented script here and forward has an alias (or declare -A) or unset in it
# run_test test_arith-for.tests - error script, looks good besides an extra error print, this is an important test
# run_test test_history2.sub - set -o history won't work here bc the history is gonna look different
Expand All @@ -2925,7 +2910,7 @@ if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
run_test test_dollar-at7.sub
run_test test_trap4.sub
# run_test test_errors9.sub - this is fine because we're just printing error statements?, dash errors here
# run_test test_dollar-star3.sub - setting IFS, also doesn't work in dash
run_test test_dollar-star3.sub
run_test test_assoc1.sub
# run_test test_alias4.sub
# run_test test_varenv6.sub
Expand All @@ -2940,7 +2925,7 @@ if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
run_test test_builtins3.sub
# run_test test_varenv7.sub
# run_test test_alias5.sub
# run_test test_dollar-star2.sub - setting IFS, also doesn't work in dash
run_test test_dollar-star2.sub
# run_test test_nameref1.sub
run_test test_errors8.sub
# run_test test_trap5.sub - I don't think this should neccesarly work since we could be catching other PaSh signals
Expand Down Expand Up @@ -2977,7 +2962,7 @@ if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
# run_test test_assoc3.sub
# run_test test_alias6.sub
# run_test test_posix2syntax.sub
# run_test test_dollar-star1.sub - setting IFS, also doesn't work in dash
run_test test_dollar-star1.sub
run_test test_getopts9.sub
run_test test_dstack2.tests
# run_test test_trap6.sub
Expand Down Expand Up @@ -3005,7 +2990,7 @@ if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
# run_test test_trap2.sub
# run_test test_alias2.sub
run_test test_assoc7.sub
# run_test test_dollar-star5.sub - setting IFS, also doesn't work in dash
run_test test_dollar-star5.sub
# run_test test_builtins4.sub
# run_test test_new-exp9.sub - local -A
run_test test_redir6.sub
Expand Down Expand Up @@ -3062,7 +3047,7 @@ if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
# run_test test_array21.sub
# run_test test_redir4.sub
# run_test test_intl2.sub
# run_test test_herestr1.sub - messing with IFS, never good
run_test test_herestr1.sub
# run_test test_builtins6.sub
run_test test_varenv2.sub
# run_test test_dollar-star7.sub
Expand Down Expand Up @@ -3145,9 +3130,9 @@ if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
run_test test_arith1.sub
# run_test test_vredir1.sub - see test_vredir.tests
# run_test test_cond.tests
# run_test test_ifs.tests - messing with IFS is a known bug in PaSh
run_test test_ifs.tests
# run_test test_varenv17.sub
# run_test test_exp3.sub - messing with IFS
run_test test_exp3.sub
run_test test_comsub-eof6.sub
# run_test test_rsh2.sub - error script
# run_test test_glob5.sub
Expand Down Expand Up @@ -3210,7 +3195,7 @@ if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
# run_test test_comsub4.sub
# run_test test_getopts10.sub
# run_test test_varenv19.sub - same local function issue
# run_test test_exp10.sub - setting IFS doesn't work
run_test test_exp10.sub
# run_test test_nquote5.tests
# run_test test_new-exp10.sub
# run_test test_array5.sub
Expand Down Expand Up @@ -3270,7 +3255,7 @@ if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
run_test test_nameref9.sub
# run_test test_assoc8.sub - declare -A here
run_test test_new-exp6.sub
# run_test test_dollar-at-star2.sub - IFS stuff again
run_test test_dollar-at-star2.sub
run_test test_exec12.sub
run_test test_source1.sub
run_test test_array10.sub
Expand Down Expand Up @@ -3308,7 +3293,7 @@ if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
# run_test test_dollar-star9.sub
# run_test test_exec11.sub - traps could catch other system stuff not expected to work
run_test test_dollar-at-star1.sub
# run_test test_new-exp5.sub - messes with IFS
run_test test_new-exp5.sub
run_test test_source2.sub
# run_test test_quotearray1.sub
# run_test test_array13.sub - declare -a
Expand Down Expand Up @@ -3360,7 +3345,7 @@ if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
run_test test_exec4.sub
# run_test test_posixexp3.sub
run_test test_errors5.sub
# run_test test_ifs1.sub - messes with IFS
run_test test_ifs1.su
run_test test_getopts7.sub
# run_test test_coproc.tests - error script
run_test test_source4.sub
Expand All @@ -3370,7 +3355,7 @@ if [ "$#" -eq 0 ] || [ "$test_mode" = "bash" ]; then
# run_test test_array15.sub
# run_test test_exportfunc.tests
# run_test test_array14.sub
# run_test test_dollar-at-star6.sub - setting IFS doesn't work
run_test test_dollar-at-star6.sub
run_test test_new-exp2.sub
run_test test_mapfile1.sub
# run_test test_source5.sub
Expand Down
Loading

0 comments on commit 3c2606b

Please sign in to comment.