From 5ae0efa848c185c1cb6d05c6818e7e97ea19a866 Mon Sep 17 00:00:00 2001 From: Justin Date: Mon, 10 Jun 2024 16:58:14 -0400 Subject: [PATCH 1/6] Rough draft of testing framework in YSH --- stdlib/testing.ysh | 246 ++++++++++++++++++++++++++++----------------- 1 file changed, 155 insertions(+), 91 deletions(-) diff --git a/stdlib/testing.ysh b/stdlib/testing.ysh index 7abce14367..04b059d926 100644 --- a/stdlib/testing.ysh +++ b/stdlib/testing.ysh @@ -1,115 +1,179 @@ -# testing.ysh -# -# Usage: -# source --builtin testing.sh -# -# func f(x) { return (x + 1) } -# -# describe foo { -# assert (43 === f(42)) -# } -# -# if is-main { -# run-tests @ARGV # --filter -# } - -module stdlib/testing || return 0 - -source --builtin args.ysh - -proc assert ( ; cond ; fail_message='default fail message') { - echo 'hi from assert' - - = cond - - # I think this might be ready now? - - var val = evalExpr(cond) - - echo - echo 'value' - = val - pp line (val) - - = fail_message - - if (val) { - echo 'OK' - } else { - var m = evalExpr(fail_message) - echo "FAIL - this is where we extract the string - $m" - } +########## +# Colors # +########## + +const _BOLD = $'\e[1m' +const _RED = $'\e[31m' +const _GREEN = $'\e[32m' +const _YELLOW = $'\e[33m' +const _PURPLE = $'\e[35m' +const _CYAN = $'\e[36m' +const _RESET = $'\e[0;0m' + +############ +# Internal # +############ + +var _test_group_stack = [] +var _tests = {} +var _test_group_depth = 0 + +var _num_fail = 0 +var _num_succ = 0 + +proc _start_test_group (; name) { + call _test_group_stack->append(name) } -proc test-assert { - var x = 42 - assert [42 === x] +proc _end_test_group { + call _test_group_stack->pop() } -proc test-expr ( ; expr ) { - echo 'expr' - pp line (expr) +proc _print_indented (msg) { + for _ in (0 .. _test_group_depth) { + printf " " + } + echo "$msg" } -proc test-named ( ; ; n=^[99] ) { - echo 'n' - pp line (n) +proc _run_test_group (; group) { + for name, elem in (group) { + if (type (elem) === "Dict") { + # It's another group. + _print_indented "${_CYAN}begin${_RESET} ${name}" + #_print_indented "${name} {" + setglobal _test_group_depth += 1; + _run_test_group (elem) + setglobal _test_group_depth -= 1; + #_print_indented "}" + _print_indented "${_CYAN}end${_RESET}" + } else { + # It's a test case. + _run_test_case (name, elem) + } + } } -# What happens when there are duplicate test IDs? -# -# Also I think filter by "$test_id/$case_id" - -proc __it (case_id ; ; ; block) { - # This uses a clean directory - echo TODO +proc _run_test_case (; name, block) { + for _ in (0 .. _test_group_depth) { + printf " " + } + printf "${_YELLOW}test${_RESET} ${name} ... " + try { + eval (block) + } + if (_status === 0) { + setglobal _num_succ += 1 + printf "${_GREEN}ok${_RESET}" + printf '\n' + } else { + setglobal _num_fail += 1 + } } -# is this accessible to users? -# It can contain a global list of things to run +########## +# Public # +########## -# Naming convention: a proc named 'describe' mutates a global named _describe? -# Or maybe _describe_list ? +proc test_group (; name ; ; block) { + _start_test_group (name) + eval (block) + _end_test_group +} -var _describe_list = [] +proc test (; name ; ; block) { + var test_group = _tests + for group_name in (_test_group_stack) { + if (not (group_name in test_group)) { + setvar test_group[group_name] = {} + } + setvar test_group = test_group[group_name] + } + setvar test_group[name] = block +} -proc describe (test_id ; ; ; block) { - echo describe - #= desc +proc assert ( ; cond LAZY ) { + var result = 1 + try { + setvar result = evalExpr(cond) + } + if (_status !== 0) { + printf '\n' + _print_indented " ${_RED}${_BOLD}EXCEPTION:${_RESET} ${_status}" + error "exception while running assertion" + } elif (not result) { + printf '\n' + _print_indented " ${_RED}${_BOLD}assertion FAILED:${_RESET} TODO" + error "assertion failed" + } +} - # TODO: - # - need append - # - need :: - # _ _describe->append(cmd) - # - # Need to clean this up - # append (_describe, cmd) # does NOT work! +proc run_tests { + setglobal _num_fail = 0 + setglobal _num_succ = 0 - call _describe_list->append(block) + _run_test_group (_tests) + setglobal _tests = {} + + var total = _num_fail + _num_succ + if (total === 0) { + echo "${_YELLOW}0 tests ran${_RESET}" + } elif (_num_fail === 0) { + echo "${_GREEN}${total} tests succeeded${_RESET}" + } else { + echo "${_RED}${_BOLD}${_num_fail} / ${total} tests failed${_RESET}" + } } -proc Args { - echo TODO +################### +# Testing Testing # +################### + +test_group ("Numbers") { + test ("numbers exist") { + assert [3] + assert [2] + assert [1] + assert [0] + } + test ("positive numbers exist") { + assert [3] + assert [2] + assert [1] + } + test ("comparisons (right)") { + assert [1 < 2] + assert [1 < 3] + } + test ("comparisons (wrong)") { + assert [1 === 1] + assert [1 > 2] + assert [1 > 3] + } + test_group ("Big Numbers") { + test ("big positive numbers") { + assert [14783482949258384725882347823841] + } + test ("big negative numbers") { + assert [-14783482949258384725882347823841] + } + } } -# Problem: this creates a global variable? -Args (&spec) { - flag --filter 'Regex of test descriptions' +test ("string test") { + assert ["abc" < "def"] } -proc run-tests { - var opt, i = parseArgs(spec, ARGV) +test_group ("Miscellaneous") { + test ("nothing") { : } + test ("something") { var x = "something" } +} - # TODO: - # - parse --filter foo, which you can use eggex for! +run_tests - for cmd in (_describe) { - # TODO: print filename and 'describe' name? - try { - eval (cmd) - } - if (_status !== 0) { - echo 'failed' - } - } +test ("good test") { + assert [true] } + +run_tests +run_tests From 729d8b98647052deecce38d6903bd13f250e94e0 Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 13 Jun 2024 16:27:50 -0400 Subject: [PATCH 2/6] Test framework improvements, and move test tests to spec/ --- spec/ysh-stdlib-testing.test.sh | 259 ++++++++++++++++++-------------- stdlib/testing.ysh | 201 +++++++++++++------------ 2 files changed, 249 insertions(+), 211 deletions(-) diff --git a/spec/ysh-stdlib-testing.test.sh b/spec/ysh-stdlib-testing.test.sh index 4e5b391adc..fb8125adc0 100644 --- a/spec/ysh-stdlib-testing.test.sh +++ b/spec/ysh-stdlib-testing.test.sh @@ -1,125 +1,158 @@ ## our_shell: ysh -## oils_failures_allowed: 5 -#### value.Expr test - positional test +#### Test framework source --builtin testing.ysh -echo 'parens' -test-expr (42 + 1) -echo +setglobal _test_use_color = false -echo 'brackets' -test-expr [42 + 1] -echo - -echo 'expr in parens' -test-expr (^[42 + 1]) -echo - -## STDOUT: -## END - -#### value.Expr test - named test - -source --builtin testing.ysh - -echo 'parens' -test-named (n=42 + 1) -echo - -echo 'brackets' -test-named [n=42 + 1] -echo - -echo 'expr in parens' -test-named (n=^[42 + 1]) -echo - -echo 'no value' -test-named -echo - -## STDOUT: -## END - -#### assert builtin - -source --builtin testing.ysh # get rid of this line later? - -var x = 42 - -# how do you get the code string here? - -assert [42 === x] - -assert [42 < x] - -#assert [42 < x; fail_message='message'] - -#assert (^[(42 < x)], fail_message='passed message') - -# BUG -assert [42 < x, fail_message='passed message'] - -## STDOUT: -## END - -#### ysh --tool test file - -cat >mytest.ysh <& 2 - return 42 -} - -describe p { - # each case changes to a clean directory? - # - # and each one is numbered? - - it 'prints to stdout and stderr' { - try { - p > out 2>& err +test_suite "some bad tests" { + test_case "bad test number one (should fail)" { + assert [1 > 0] + assert [1 > 1] + assert [1 > 2] } - assert (_status === 42) - - cat out - cat err - - # Oh man the here docs are still useful here because of 'diff' interface - # Multiline strings don't quite do it - - diff out - <<< ''' - STDOUT - ''' + test_case "bad test number two (should error)" { + assert ["Superman" > "Batman"] + } +} - diff err - <<< ''' - STDERR - ''' - } +test_case "one good test case" { + assert [1 === 1] } +run_tests + ## STDOUT: -TODO +begin some bad tests + test bad test number one (should fail) ... + assertion FAILED: TODO + test bad test number two (should error) ... + EXCEPTION: 3 +end +test one good test case ... ok +2 / 3 tests failed ## END + +# #### value.Expr test - positional test +# +# source --builtin testing.ysh +# +# echo 'parens' +# test-expr (42 + 1) +# echo +# +# echo 'brackets' +# test-expr [42 + 1] +# echo +# +# echo 'expr in parens' +# test-expr (^[42 + 1]) +# echo +# +# ## STDOUT: +# ## END +# +# #### value.Expr test - named test +# +# source --builtin testing.ysh +# +# echo 'parens' +# test-named (n=42 + 1) +# echo +# +# echo 'brackets' +# test-named [n=42 + 1] +# echo +# +# echo 'expr in parens' +# test-named (n=^[42 + 1]) +# echo +# +# echo 'no value' +# test-named +# echo +# +# ## STDOUT: +# ## END +# +# #### assert builtin +# +# source --builtin testing.ysh # get rid of this line later? +# +# var x = 42 +# +# # how do you get the code string here? +# +# assert [42 === x] +# +# assert [42 < x] +# +# #assert [42 < x; fail_message='message'] +# +# #assert (^[(42 < x)], fail_message='passed message') +# +# # BUG +# assert [42 < x, fail_message='passed message'] +# +# ## STDOUT: +# ## END +# +# #### ysh --tool test file +# +# cat >mytest.ysh <& 2 +# return 42 +# } +# +# describe p { +# # each case changes to a clean directory? +# # +# # and each one is numbered? +# +# it 'prints to stdout and stderr' { +# try { +# p > out 2>& err +# } +# assert (_status === 42) +# +# cat out +# cat err +# +# # Oh man the here docs are still useful here because of 'diff' interface +# # Multiline strings don't quite do it +# +# diff out - <<< ''' +# STDOUT +# ''' +# +# diff err - <<< ''' +# STDERR +# ''' +# } +# } +# +# ## STDOUT: +# TODO +# ## END diff --git a/stdlib/testing.ysh b/stdlib/testing.ysh index 04b059d926..ea0360d32c 100644 --- a/stdlib/testing.ysh +++ b/stdlib/testing.ysh @@ -1,3 +1,15 @@ +# TODO: +# - [x] Renaming a bit for "privacy" +# - [x] Renaming for public interface: test-case, test-suite, assert (subject to change) +# - [x] Make test-case and test-suite take a word arg so they don't need parens +# - [x] error if you try to re-enter run-tests +# - [ ] testing testing: assert fail, exn in assert, exn outside of assert +# - [ ] check for both kinds of exceptions (_status and _error) +# - [ ] tests for tests +# - [ ] --tool test support +# - [ ] Turn the assert expression into a string, to print if it fails +# - [ ] Merge _status and _error + ########## # Colors # ########## @@ -10,43 +22,74 @@ const _PURPLE = $'\e[35m' const _CYAN = $'\e[36m' const _RESET = $'\e[0;0m' +func _red(text) { + if (_test_use_color) { + return ("${_BOLD}${_RED}${text}${_RESET}") + } else { + return (text) + } +} +func _yellow(text) { + if (_test_use_color) { + return ("${_YELLOW}${text}${_RESET}") + } else { + return (text) + } +} +func _green(text) { + if (_test_use_color) { + return ("${_YELLOW}${text}${_RESET}") + } else { + return (text) + } +} +func _cyan(text) { + if (_test_use_color) { + return ("${_YELLOW}${text}${_RESET}") + } else { + return (text) + } +} + ############ # Internal # ############ -var _test_group_stack = [] +var _testing_in_progress = false +var _test_suite_depth = 0 +var _test_suite_stack = [] var _tests = {} -var _test_group_depth = 0 -var _num_fail = 0 -var _num_succ = 0 +var _num_test_fail = 0 +var _num_test_succ = 0 +var _test_use_color = true -proc _start_test_group (; name) { - call _test_group_stack->append(name) +proc _start_test_suite (; name) { + call _test_suite_stack->append(name) } -proc _end_test_group { - call _test_group_stack->pop() +proc _end_test_suite { + call _test_suite_stack->pop() } -proc _print_indented (msg) { - for _ in (0 .. _test_group_depth) { +proc _test_print_indented (msg) { + for _ in (0 .. _test_suite_depth) { printf " " } echo "$msg" } -proc _run_test_group (; group) { - for name, elem in (group) { +proc _run_test_suite (; suite) { + for name, elem in (suite) { if (type (elem) === "Dict") { - # It's another group. - _print_indented "${_CYAN}begin${_RESET} ${name}" - #_print_indented "${name} {" - setglobal _test_group_depth += 1; - _run_test_group (elem) - setglobal _test_group_depth -= 1; - #_print_indented "}" - _print_indented "${_CYAN}end${_RESET}" + # It's another suite. + var begin = _cyan("begin") + _test_print_indented "$begin $name" + setglobal _test_suite_depth += 1; + _run_test_suite (elem) + setglobal _test_suite_depth -= 1; + var end = _cyan("end") + _test_print_indented "$end" } else { # It's a test case. _run_test_case (name, elem) @@ -55,19 +98,21 @@ proc _run_test_group (; group) { } proc _run_test_case (; name, block) { - for _ in (0 .. _test_group_depth) { + for _ in (0 .. _test_suite_depth) { printf " " } - printf "${_YELLOW}test${_RESET} ${name} ... " + var test = _yellow("test") + printf "$test $name ... " try { eval (block) } if (_status === 0) { - setglobal _num_succ += 1 - printf "${_GREEN}ok${_RESET}" + setglobal _num_test_succ += 1 + var ok = _green("ok") + printf "$ok" printf '\n' } else { - setglobal _num_fail += 1 + setglobal _num_test_fail += 1 } } @@ -75,21 +120,21 @@ proc _run_test_case (; name, block) { # Public # ########## -proc test_group (; name ; ; block) { - _start_test_group (name) +proc test_suite (name ; ; ; block) { + _start_test_suite (name) eval (block) - _end_test_group + _end_test_suite } -proc test (; name ; ; block) { - var test_group = _tests - for group_name in (_test_group_stack) { - if (not (group_name in test_group)) { - setvar test_group[group_name] = {} +proc test_case (name ; ; ; block) { + var test_suite = _tests + for suite_name in (_test_suite_stack) { + if (not (suite_name in test_suite)) { + setvar test_suite[suite_name] = {} } - setvar test_group = test_group[group_name] + setvar test_suite = test_suite[suite_name] } - setvar test_group[name] = block + setvar test_suite[name] = block } proc assert ( ; cond LAZY ) { @@ -99,81 +144,41 @@ proc assert ( ; cond LAZY ) { } if (_status !== 0) { printf '\n' - _print_indented " ${_RED}${_BOLD}EXCEPTION:${_RESET} ${_status}" + var exn = _red("EXCEPTION:") + _test_print_indented " $exn $_status" error "exception while running assertion" } elif (not result) { printf '\n' - _print_indented " ${_RED}${_BOLD}assertion FAILED:${_RESET} TODO" + var fail = _red("assertion FAILED:") + _test_print_indented " $fail TODO" error "assertion failed" } } proc run_tests { - setglobal _num_fail = 0 - setglobal _num_succ = 0 + if (_testing_in_progress) { + error "Cannot run tests while testing is already in progress" + } + setglobal _testing_in_progress = true + setglobal _num_test_fail = 0 + setglobal _num_test_succ = 0 - _run_test_group (_tests) + _run_test_suite (_tests) setglobal _tests = {} + setglobal _testing_in_progress = false - var total = _num_fail + _num_succ + var total = _num_test_fail + _num_test_succ if (total === 0) { - echo "${_YELLOW}0 tests ran${_RESET}" - } elif (_num_fail === 0) { - echo "${_GREEN}${total} tests succeeded${_RESET}" + var na = _yellow("0 tests ran") + printf "$na" + printf '\n' + } elif (_num_test_fail === 0) { + var success = _green("$total tests succeeded") + printf "$success" + printf '\n' } else { - echo "${_RED}${_BOLD}${_num_fail} / ${total} tests failed${_RESET}" - } -} - -################### -# Testing Testing # -################### - -test_group ("Numbers") { - test ("numbers exist") { - assert [3] - assert [2] - assert [1] - assert [0] - } - test ("positive numbers exist") { - assert [3] - assert [2] - assert [1] - } - test ("comparisons (right)") { - assert [1 < 2] - assert [1 < 3] - } - test ("comparisons (wrong)") { - assert [1 === 1] - assert [1 > 2] - assert [1 > 3] - } - test_group ("Big Numbers") { - test ("big positive numbers") { - assert [14783482949258384725882347823841] - } - test ("big negative numbers") { - assert [-14783482949258384725882347823841] - } + var failure = _red("$_num_test_fail / $total tests failed") + printf "$failure" + printf '\n' } } - -test ("string test") { - assert ["abc" < "def"] -} - -test_group ("Miscellaneous") { - test ("nothing") { : } - test ("something") { var x = "something" } -} - -run_tests - -test ("good test") { - assert [true] -} - -run_tests -run_tests From b738f3a9151887014adba356b645770f90cbb1b0 Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 13 Jun 2024 17:31:29 -0400 Subject: [PATCH 3/6] Correctly handle the three ways that a test case can fail, and test them --- spec/ysh-stdlib-testing.test.sh | 25 +++++++++++------ stdlib/testing.ysh | 49 ++++++++++++++++++++++----------- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/spec/ysh-stdlib-testing.test.sh b/spec/ysh-stdlib-testing.test.sh index fb8125adc0..9cb222f5b1 100644 --- a/spec/ysh-stdlib-testing.test.sh +++ b/spec/ysh-stdlib-testing.test.sh @@ -6,15 +6,20 @@ source --builtin testing.ysh setglobal _test_use_color = false -test_suite "some bad tests" { - test_case "bad test number one (should fail)" { +test_suite "three kinds of failures" { + test_case "the assertion is false" { assert [1 > 0] assert [1 > 1] assert [1 > 2] } - test_case "bad test number two (should error)" { + + test_case "there is an exception while evaluating the assertion" { assert ["Superman" > "Batman"] } + + test_case "there is an exception outside of any assertion" { + error "oops!" + } } test_case "one good test case" { @@ -24,14 +29,16 @@ test_case "one good test case" { run_tests ## STDOUT: -begin some bad tests - test bad test number one (should fail) ... - assertion FAILED: TODO - test bad test number two (should error) ... - EXCEPTION: 3 +begin three kinds of failures + test the assertion is false ... + assertion FAILED + test there is an exception while evaluating the assertion ... + assertion ERRORED: 10 + test there is an exception outside of any assertion ... + ERROR: 10 end test one good test case ... ok -2 / 3 tests failed +3 / 4 tests failed ## END # #### value.Expr test - positional test diff --git a/stdlib/testing.ysh b/stdlib/testing.ysh index ea0360d32c..92cfb69c3a 100644 --- a/stdlib/testing.ysh +++ b/stdlib/testing.ysh @@ -1,14 +1,18 @@ # TODO: # - [x] Renaming a bit for "privacy" # - [x] Renaming for public interface: test-case, test-suite, assert (subject to change) +# - [ ] Rename to use "-"s for procs. # - [x] Make test-case and test-suite take a word arg so they don't need parens # - [x] error if you try to re-enter run-tests -# - [ ] testing testing: assert fail, exn in assert, exn outside of assert -# - [ ] check for both kinds of exceptions (_status and _error) +# - [x] testing testing: assert fail, exn in assert, exn outside of assert +# - [-] check for both kinds of exceptions (_status and _error) # - [ ] tests for tests # - [ ] --tool test support # - [ ] Turn the assert expression into a string, to print if it fails -# - [ ] Merge _status and _error +# - [ ] Use ctx instead of globals, wherever possible +# - [ ] Merge _status and _error. Every exception should set _error, with the +# status stored on _error.status. Use _error instead of the icky global +# _assertion_result ########## # Colors # @@ -64,6 +68,8 @@ var _num_test_fail = 0 var _num_test_succ = 0 var _test_use_color = true +var _assertion_result = 0 # 0: no assert, 1: failed, 2: errored + proc _start_test_suite (; name) { call _test_suite_stack->append(name) } @@ -98,21 +104,36 @@ proc _run_test_suite (; suite) { } proc _run_test_case (; name, block) { + var test = _yellow("test") for _ in (0 .. _test_suite_depth) { printf " " } - var test = _yellow("test") - printf "$test $name ... " + printf "$test $name ..." + setglobal _assertion_result = 0 try { eval (block) } if (_status === 0) { setglobal _num_test_succ += 1 - var ok = _green("ok") + var ok = _green(" ok") printf "$ok" printf '\n' } else { setglobal _num_test_fail += 1 + printf '\n' + if (_assertion_result === 1) { + # An assertion failed. + var fail = _red("assertion FAILED") + _test_print_indented " $fail" + } elif (_assertion_result === 2) { + # There was an exception while evaluating an assertion. + var exn_in_assert = _red("assertion ERRORED:") + _test_print_indented " $exn_in_assert $_status" + } else { + # There was an exception in the test case outside of any `assert`. + var exn = _red("ERROR:") + _test_print_indented " $exn $_status" + } } } @@ -138,19 +159,15 @@ proc test_case (name ; ; ; block) { } proc assert ( ; cond LAZY ) { - var result = 1 + var success try { - setvar result = evalExpr(cond) + setvar success = evalExpr(cond) } if (_status !== 0) { - printf '\n' - var exn = _red("EXCEPTION:") - _test_print_indented " $exn $_status" - error "exception while running assertion" - } elif (not result) { - printf '\n' - var fail = _red("assertion FAILED:") - _test_print_indented " $fail TODO" + setglobal _assertion_result = 2 + error "exception in assertion" + } elif (not success) { + setglobal _assertion_result = 1 error "assertion failed" } } From a6e9c811cf6c0268c95236e97f93b7e99020f8c0 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 18 Jun 2024 14:29:17 -0400 Subject: [PATCH 4/6] Tests for tests --- spec/ysh-stdlib-testing.test.sh | 143 ++++++++------------------------ stdlib/testing.ysh | 2 +- 2 files changed, 36 insertions(+), 109 deletions(-) diff --git a/spec/ysh-stdlib-testing.test.sh b/spec/ysh-stdlib-testing.test.sh index 9cb222f5b1..ed28df237e 100644 --- a/spec/ysh-stdlib-testing.test.sh +++ b/spec/ysh-stdlib-testing.test.sh @@ -3,10 +3,9 @@ #### Test framework source --builtin testing.ysh - setglobal _test_use_color = false -test_suite "three kinds of failures" { +test_suite "three bad tests" { test_case "the assertion is false" { assert [1 > 0] assert [1 > 1] @@ -24,12 +23,13 @@ test_suite "three kinds of failures" { test_case "one good test case" { assert [1 === 1] + assert [2 === 2] } run_tests ## STDOUT: -begin three kinds of failures +begin three bad tests test the assertion is false ... assertion FAILED test there is an exception while evaluating the assertion ... @@ -41,70 +41,38 @@ test one good test case ... ok 3 / 4 tests failed ## END -# #### value.Expr test - positional test -# -# source --builtin testing.ysh -# -# echo 'parens' -# test-expr (42 + 1) -# echo -# -# echo 'brackets' -# test-expr [42 + 1] -# echo -# -# echo 'expr in parens' -# test-expr (^[42 + 1]) -# echo -# -# ## STDOUT: -# ## END -# -# #### value.Expr test - named test -# -# source --builtin testing.ysh -# -# echo 'parens' -# test-named (n=42 + 1) -# echo -# -# echo 'brackets' -# test-named [n=42 + 1] -# echo -# -# echo 'expr in parens' -# test-named (n=^[42 + 1]) -# echo -# -# echo 'no value' -# test-named -# echo -# -# ## STDOUT: -# ## END -# -# #### assert builtin -# -# source --builtin testing.ysh # get rid of this line later? -# -# var x = 42 -# -# # how do you get the code string here? -# -# assert [42 === x] -# -# assert [42 < x] -# -# #assert [42 < x; fail_message='message'] -# -# #assert (^[(42 < x)], fail_message='passed message') -# -# # BUG -# assert [42 < x, fail_message='passed message'] -# -# ## STDOUT: -# ## END -# +#### Stdout and stderr + +source --builtin testing.ysh +setglobal _test_use_color = false + +proc p { + echo STDOUT + echo STDERR >& 2 + return 42 +} + +test_case "that it prints to stdout and stderr" { + # each case changes to a clean directory? + # + # and each one is numbered? + + try { + p > out 2> err + } + + assert [_status === 42] + assert [$(mytest.ysh <& 2 -# return 42 -# } -# -# describe p { -# # each case changes to a clean directory? -# # -# # and each one is numbered? -# -# it 'prints to stdout and stderr' { -# try { -# p > out 2>& err -# } -# assert (_status === 42) -# -# cat out -# cat err -# -# # Oh man the here docs are still useful here because of 'diff' interface -# # Multiline strings don't quite do it -# -# diff out - <<< ''' -# STDOUT -# ''' -# -# diff err - <<< ''' -# STDERR -# ''' -# } -# } -# -# ## STDOUT: -# TODO -# ## END diff --git a/stdlib/testing.ysh b/stdlib/testing.ysh index 92cfb69c3a..6479a1af69 100644 --- a/stdlib/testing.ysh +++ b/stdlib/testing.ysh @@ -6,7 +6,7 @@ # - [x] error if you try to re-enter run-tests # - [x] testing testing: assert fail, exn in assert, exn outside of assert # - [-] check for both kinds of exceptions (_status and _error) -# - [ ] tests for tests +# - [x] tests for tests # - [ ] --tool test support # - [ ] Turn the assert expression into a string, to print if it fails # - [ ] Use ctx instead of globals, wherever possible From 4c738ee6d1038e4ec52313be450b927157088e73 Mon Sep 17 00:00:00 2001 From: Justin Date: Tue, 18 Jun 2024 17:16:46 -0400 Subject: [PATCH 5/6] Rename procs to use "-", add more test tests. --- spec/ysh-stdlib-testing.test.sh | 56 +++++++++++++++++++++++++++------ stdlib/testing.ysh | 38 +++++++++++----------- 2 files changed, 66 insertions(+), 28 deletions(-) diff --git a/spec/ysh-stdlib-testing.test.sh b/spec/ysh-stdlib-testing.test.sh index ed28df237e..c8a0c2f093 100644 --- a/spec/ysh-stdlib-testing.test.sh +++ b/spec/ysh-stdlib-testing.test.sh @@ -5,28 +5,28 @@ source --builtin testing.ysh setglobal _test_use_color = false -test_suite "three bad tests" { - test_case "the assertion is false" { +test-suite "three bad tests" { + test-case "the assertion is false" { assert [1 > 0] assert [1 > 1] assert [1 > 2] } - test_case "there is an exception while evaluating the assertion" { + test-case "there is an exception while evaluating the assertion" { assert ["Superman" > "Batman"] } - test_case "there is an exception outside of any assertion" { + test-case "there is an exception outside of any assertion" { error "oops!" } } -test_case "one good test case" { +test-case "one good test case" { assert [1 === 1] assert [2 === 2] } -run_tests +run-tests ## STDOUT: begin three bad tests @@ -41,7 +41,45 @@ test one good test case ... ok 3 / 4 tests failed ## END -#### Stdout and stderr +#### Test framework: nested test suites + +source --builtin testing.ysh +setglobal _test_use_color = false + +test-case "A" { : } +test-suite "outer test suite" { + test-case "B" { : } + test-suite "first inner test suite" { + test-case "C" { : } + } + test-case "D" { : } + test-suite "second inner test suite" { + test-case "E" { : } + } + test-case "F" { : } +} +test-case "G" { : } + +run-tests + +## STDOUT: +test A ... ok +begin outer test suite + test B ... ok + begin first inner test suite + test C ... ok + end + test D ... ok + begin second inner test suite + test E ... ok + end + test F ... ok +end +test G ... ok +7 tests succeeded +## END + +#### Test framework: stdout and stderr source --builtin testing.ysh setglobal _test_use_color = false @@ -52,7 +90,7 @@ proc p { return 42 } -test_case "that it prints to stdout and stderr" { +test-case "that it prints to stdout and stderr" { # each case changes to a clean directory? # # and each one is numbered? @@ -66,7 +104,7 @@ test_case "that it prints to stdout and stderr" { assert [$(append(name) } -proc _end_test_suite { +proc _end-test-suite { call _test_suite_stack->pop() } -proc _test_print_indented (msg) { +proc _test-print-indented (msg) { for _ in (0 .. _test_suite_depth) { printf " " } echo "$msg" } -proc _run_test_suite (; suite) { +proc _run-test-suite (; suite) { for name, elem in (suite) { if (type (elem) === "Dict") { # It's another suite. var begin = _cyan("begin") - _test_print_indented "$begin $name" + _test-print-indented "$begin $name" setglobal _test_suite_depth += 1; - _run_test_suite (elem) + _run-test-suite (elem) setglobal _test_suite_depth -= 1; var end = _cyan("end") - _test_print_indented "$end" + _test-print-indented "$end" } else { # It's a test case. - _run_test_case (name, elem) + _run-test-case (name, elem) } } } -proc _run_test_case (; name, block) { +proc _run-test-case (; name, block) { var test = _yellow("test") for _ in (0 .. _test_suite_depth) { printf " " @@ -124,15 +124,15 @@ proc _run_test_case (; name, block) { if (_assertion_result === 1) { # An assertion failed. var fail = _red("assertion FAILED") - _test_print_indented " $fail" + _test-print-indented " $fail" } elif (_assertion_result === 2) { # There was an exception while evaluating an assertion. var exn_in_assert = _red("assertion ERRORED:") - _test_print_indented " $exn_in_assert $_status" + _test-print-indented " $exn_in_assert $_status" } else { # There was an exception in the test case outside of any `assert`. var exn = _red("ERROR:") - _test_print_indented " $exn $_status" + _test-print-indented " $exn $_status" } } } @@ -141,13 +141,13 @@ proc _run_test_case (; name, block) { # Public # ########## -proc test_suite (name ; ; ; block) { - _start_test_suite (name) +proc test-suite (name ; ; ; block) { + _start-test-suite (name) eval (block) - _end_test_suite + _end-test-suite } -proc test_case (name ; ; ; block) { +proc test-case (name ; ; ; block) { var test_suite = _tests for suite_name in (_test_suite_stack) { if (not (suite_name in test_suite)) { @@ -172,7 +172,7 @@ proc assert ( ; cond LAZY ) { } } -proc run_tests { +proc run-tests { if (_testing_in_progress) { error "Cannot run tests while testing is already in progress" } @@ -180,7 +180,7 @@ proc run_tests { setglobal _num_test_fail = 0 setglobal _num_test_succ = 0 - _run_test_suite (_tests) + _run-test-suite (_tests) setglobal _tests = {} setglobal _testing_in_progress = false From 29499406e6dbf8672239ec9122e2a9228c3e0ada Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 20 Jun 2024 14:30:38 -0400 Subject: [PATCH 6/6] minor: update todo list --- stdlib/testing.ysh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stdlib/testing.ysh b/stdlib/testing.ysh index 2a0ad6b4ab..b154cfe85a 100644 --- a/stdlib/testing.ysh +++ b/stdlib/testing.ysh @@ -1,12 +1,13 @@ # TODO: # - [x] Renaming a bit for "privacy" # - [x] Renaming for public interface: test-case, test-suite, assert (subject to change) -# - [ ] Rename to use "-"s for procs. +# - [x] Rename to use "-"s for procs. # - [x] Make test-case and test-suite take a word arg so they don't need parens # - [x] error if you try to re-enter run-tests # - [x] testing testing: assert fail, exn in assert, exn outside of assert # - [-] check for both kinds of exceptions (_status and _error) # - [x] tests for tests +# - [ ] question: naming for variables? # - [ ] --tool test support (& spec test case for it) # - [ ] Turn the assert expression into a string, to print if it fails # - [ ] Use ctx instead of globals, wherever possible