Skip to content

Commit

Permalink
#2 & #7: report skipped tests via the reporter interface
Browse files Browse the repository at this point in the history
  • Loading branch information
@Kosta-Github
Kosta-Github committed Mar 23, 2014
1 parent 46973ea commit 2c13fb1
Show file tree
Hide file tree
Showing 9 changed files with 112 additions and 100 deletions.
6 changes: 3 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@ Feature set:
- no external dependencies
- header-only
- thread-safe
- provides capabilities for mocking classes and interfaces
- can use custom reporters
- supports tags for test case filtering
- can use custom reporters (provided: simple ide/console reporter, JUnit XML reporter)
- provides capabilities for mocking classes and interfaces (planned)

status: alpha
=============
Expand All @@ -22,4 +22,4 @@ Based on the following ideas and frameworks:

dependencies
============
- [cmake](http://cmake.org/) (only required for building and executing the unit test suite)
- [cmake](http://cmake.org/) (only required for building and executing the self-test unit test suite)
75 changes: 35 additions & 40 deletions cute/context.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,60 +48,55 @@ namespace cute {

for(auto&& test : tests) {
eval.current_test = &test;
++eval.test_cases;

auto rep = test_result(test);

if(detail::skip_test(test.tags, incl_tags, excl_tags)) {
++eval.test_cases_skipped;
// rep.result = result_type::skip;
// eval.test_results.emplace_back(rep);
continue;
}

auto const time_start = detail::time_now();

try {
auto const count_start = eval.checks_performed.load();

--eval.checks_performed; // decr by one since CUTE_DETAIL_ASSERT() below will increment it again
CUTE_DETAIL_ASSERT(((void)test.test_case(), true), test.file, test.line, "", cute::captures(), cute::captures());

auto const count_end = eval.checks_performed.load();
if(count_start == count_end) {
throw cute::exception("no check performed in test case", test.file, test.line, "");
}

// ensure that the temp folder can be cleared and that no file locks exists after the test case
if(!eval.delete_temp_folder()) {
throw cute::exception("could not cleanup temp folder", test.file, test.line, "");
}
auto skip = detail::skip_test(test.tags, incl_tags, excl_tags);

if(!eval.exceptions.empty()) {
throw eval.exceptions.front();
auto rep = test_result(test);
rep.result = (skip ? result_type::skip : result_type::pass);

if(!skip) {
auto const time_start = detail::time_now();

try {
auto const count_start = eval.checks_performed.load();

--eval.checks_performed; // decr by one since CUTE_DETAIL_ASSERT() below will increment it again
CUTE_DETAIL_ASSERT((static_cast<void>(test.test_case()), true), test.file, test.line, "", cute::captures(), cute::captures());

auto const count_end = eval.checks_performed.load();
if(count_start == count_end) {
eval.add_exception(cute::exception("no check performed in test case", test.file, test.line, ""));
}

// ensure that the temp folder can be cleared and that no file locks exists after the test case
if(!eval.delete_temp_folder()) {
eval.add_exception(cute::exception("could not cleanup temp folder", test.file, test.line, ""));
}
} catch(cute::exception const& ex) {
// nothing to do here anymore
} catch(std::exception const& ex) {
eval.add_exception(cute::exception("unexpected exception", test.file, test.line, ex.what()), false);
} catch(...) {
eval.add_exception(cute::exception("unexpected exception", test.file, test.line, ""), false);
}

++eval.test_cases_passed;
} catch(cute::exception const& ex) {
rep.result = result_type::fail;
rep.exceptions.emplace_back(ex);

// ensure that the temp folder gets also cleared even in case of a test failure
eval.delete_temp_folder();

++eval.test_cases_failed;
}
auto const time_end = detail::time_now();
rep.duration_ms = detail::time_diff_ms(time_start, time_end);

auto const time_end = detail::time_now();
rep.duration_ms = detail::time_diff_ms(time_start, time_end);
if(!eval.exceptions.empty()) {
rep.result = result_type::fail;
rep.exceptions = std::move(eval.exceptions);
}
}

eval.test_results.emplace_back(rep);

for(auto&& reporter : reporters) {
if(reporter) { reporter(rep); }
}

eval.exceptions.clear();
}

eval.current_test = nullptr;
Expand Down
11 changes: 2 additions & 9 deletions cute/detail/eval_context.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ namespace cute {
struct eval_context {
eval_context() :
reporters(nullptr), current_test(nullptr),
test_cases(0), test_cases_passed(0),
test_cases_failed(0), test_cases_skipped(0),
checks_performed(0), duration_ms(0),
m_previous(g_current)
{
Expand All @@ -36,12 +34,8 @@ namespace cute {

test const* current_test;

std::atomic<std::size_t> test_cases;
std::atomic<std::size_t> test_cases_passed;
std::atomic<std::size_t> test_cases_failed;
std::atomic<std::size_t> test_cases_skipped;
std::atomic<std::size_t> checks_performed;
std::atomic<std::size_t> duration_ms;
std::size_t duration_ms;

std::mutex exceptions_mutex;
std::vector<exception> exceptions;
Expand Down Expand Up @@ -73,8 +67,7 @@ namespace cute {

inline test_suite_result result() {
return test_suite_result(
test_cases, test_cases_passed, test_cases_failed, test_cases_skipped,
checks_performed, duration_ms, std::move(test_results)
std::move(test_results), checks_performed, duration_ms
);
}

Expand Down
2 changes: 1 addition & 1 deletion cute/detail/macros_impl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
try { \
++cute::detail::eval_context::current().checks_performed; \
if(!(EXPR_EVAL)) { \
cute::detail::eval_context::current().add_exception(cute::exception("", FILE, LINE, EXPR_TEXT, CAPS1, CAP2)); \
cute::detail::eval_context::current().add_exception(cute::exception(EXPR_TEXT, FILE, LINE, "", CAPS1, CAP2)); \
} \
} catch(cute::exception const& ex) { \
cute::detail::eval_context::current().add_exception(ex); \
Expand Down
25 changes: 13 additions & 12 deletions cute/reporters/reporter_ide.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ namespace cute {
auto type = std::string();
switch(res.result) {
case result_type::pass: type = "pass: "; break;
case result_type::skip: type = "skip: "; break;
case result_type::fail: type = "error: "; break;
case result_type::fatal: type = "fatal: "; break;
default: assert(false);
Expand All @@ -39,22 +40,22 @@ namespace cute {
auto test_header = detail::ide_make_file_line_string(res.test.file, res.test.line) + type;
os << test_header << res.test.name << std::endl;

if(res.result != result_type::pass) {
for(auto&& ex : res.exceptions) {
auto ex_header = detail::ide_make_file_line_string(ex.file, ex.line) + type;
if(res.result != result_type::skip) {
os << test_header << " duration: " << res.duration_ms << " ms" << std::endl;
}

for(auto&& ex : res.exceptions) {
auto ex_header = detail::ide_make_file_line_string(ex.file, ex.line) + type;

os << ex_header << " reason: " << ex.what() << std::endl;
if(!ex.expr.empty()) { os << ex_header << " expression: " << ex.expr << std::endl; }
os << ex_header << " reason: " << ex.what() << std::endl;
if(!ex.expr.empty()) { os << ex_header << " expression: " << ex.expr << std::endl; }

for(auto&& c : ex.captures.list) {
os << ex_header << " with: " << c.name;
if(c.name != c.value) { os << " => " << c.value; }
os << std::endl;
}
for(auto&& c : ex.captures.list) {
os << ex_header << " with: " << c.name;
if(c.name != c.value) { os << " => " << c.value; }
os << std::endl;
}
}

os << test_header << " duration: " << res.duration_ms << " ms" << std::endl;
}

} // namespace cute
40 changes: 24 additions & 16 deletions cute/reporters/reporter_junit.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,31 +40,39 @@ namespace cute {
out << "classname=\"" << xml_encode(name) << "\" ";
out << "name=\"" << xml_encode(test.test.name) << "\" ";
out << "time=\"" << (test.duration_ms / 1000.0f) << "\" ";

if(test.result == result_type::pass) {
out << "/>" << std::endl;
return;
}
out << ">" << std::endl;

std::string type;
switch(test.result) {
case result_type::pass: assert(false); // already handled above
case result_type::pass: type = "pass"; break;
case result_type::skip: type = "skip"; break;
case result_type::fail: type = "fail"; break;
case result_type::fatal: type = "fatal"; break;
default: assert(false);
}

out << ">" << std::endl;

for(auto&& ex : test.exceptions) {
out << " <error ";
out << "type=\"" << type << "\" ";
out << "message=\"" << xml_encode(ex.expr) << "\" ";
out << ">" << std::endl;
out << "at " << xml_encode(ex.file) << ":" << ex.line << std::endl;
out << " </error>" << std::endl;
out << " </testcase>" << std::endl;
switch(test.result) {
case result_type::pass: break;
case result_type::skip: {
out << " <skipped/>" << std::endl;
break;
}
case result_type::fail:
case result_type::fatal: {
for(auto&& ex : test.exceptions) {
out << " <error ";
out << "type=\"" << type << "\" ";
out << "message=\"" << xml_encode(ex.what()) << "\" ";
out << ">" << std::endl;
out << "at " << xml_encode(ex.file) << ":" << ex.line << std::endl;
out << " </error>" << std::endl;
}
break;
}
default: assert(false);
}

out << " </testcase>" << std::endl;
}

inline void junit_write_lead_out(
Expand Down
1 change: 1 addition & 0 deletions cute/test_result.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ namespace cute {

enum class result_type {
pass,
skip,
fail,
fatal
};
Expand Down
33 changes: 20 additions & 13 deletions cute/test_suite_result.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,28 +14,35 @@
namespace cute {

struct test_suite_result {
std::size_t const test_cases;
std::vector<test_result> const test_results;
std::size_t const test_cases_passed;
std::size_t const test_cases_failed;
std::size_t const test_cases_skipped;
std::size_t const test_cases_failed;
std::size_t const checks_performed;
std::size_t const duration_ms; // milliseconds
std::vector<test_result> const test_results;

inline test_suite_result(
std::size_t test_cases_,
std::size_t test_cases_passed_,
std::size_t test_cases_failed_,
std::size_t test_cases_skipped_,
std::vector<test_result> test_results_,
std::size_t checks_performed_,
std::size_t duration_ms_,
std::vector<test_result> test_results_
std::size_t duration_ms_
) :
test_cases(test_cases_), test_cases_passed(test_cases_passed_),
test_cases_failed(test_cases_failed_), test_cases_skipped(test_cases_skipped_),
checks_performed(checks_performed_), duration_ms(duration_ms_),
test_results(std::move(test_results_))
test_results(std::move(test_results_)),
test_cases_passed(0), test_cases_skipped(0), test_cases_failed(0),
checks_performed(checks_performed_),
duration_ms(duration_ms_)
{
auto& passed = const_cast<std::size_t&>(test_cases_passed);
auto& skipped = const_cast<std::size_t&>(test_cases_skipped);
auto& failed = const_cast<std::size_t&>(test_cases_failed);
for(auto&& r : test_results) {
switch(r.result) {
case result_type::pass: ++passed; break;
case result_type::skip: ++skipped; break;
case result_type::fail: ++failed; break;
case result_type::fatal: ++failed; break;
default: assert(false);
}
}
}
};

Expand Down
19 changes: 13 additions & 6 deletions test/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,25 +15,32 @@ int main(int argc, char* argv[]) {
auto pass_ctx = cute::context();
pass_ctx.include_tags = "pass";
pass_ctx.reporters.emplace_back([&](cute::test_result const& res) {
cute::reporter_ide((res.result == cute::result_type::pass) ? std::cout : std::cerr, res);
cute::reporter_ide(std::cout, res);
});
auto pass_res = pass_ctx.run();

auto fail_ctx = cute::context();
fail_ctx.include_tags = "fail";
fail_ctx.reporters.emplace_back([&](cute::test_result const& res) {
auto r = res; r.result = (res.result == cute::result_type::pass) ? cute::result_type::fail : cute::result_type::pass;
cute::reporter_ide((res.result == cute::result_type::pass) ? std::cout : std::cerr, r);
auto r = res;
switch(r.result) {
case cute::result_type::pass: r.result = cute::result_type::fail; break;
case cute::result_type::skip: break;
case cute::result_type::fail: r.result = cute::result_type::pass; break;
case cute::result_type::fatal: break;
default: assert(false);
}
cute::reporter_ide(std::cout, r);
});
auto fail_res = fail_ctx.run();

// for this unit test in which we check the correct behavior for
// passing and failing test cases we need to combine both results
// by inverting the "failed" cases...
auto tests_all = pass_res.test_cases;
auto tests_all = pass_res.test_results.size();
auto tests_passed = pass_res.test_cases_passed + fail_res.test_cases_failed;
auto tests_failed = pass_res.test_cases_failed + fail_res.test_cases_passed;
auto tests_skipped = pass_res.test_cases_skipped + fail_res.test_cases_skipped - tests_all;
auto tests_failed = pass_res.test_cases_failed + fail_res.test_cases_passed;
auto checks_performed = pass_res.checks_performed + fail_res.checks_performed;
auto duration_ms = pass_res.duration_ms + fail_res.duration_ms;

Expand All @@ -44,8 +51,8 @@ int main(int argc, char* argv[]) {
out << std::endl;
out << "test summary:" << std::endl;
out << tests_passed << " out of " << tests_all << " tests passed." << std::endl;
out << tests_failed << " out of " << tests_all << " tests failed." << std::endl;
out << tests_skipped << " out of " << tests_all << " tests skipped." << std::endl;
out << tests_failed << " out of " << tests_all << " tests failed." << std::endl;
out << checks_performed << " checks performed." << std::endl;
out << duration_ms << " ms used for the whole test suite." << std::endl;

Expand Down

0 comments on commit 2c13fb1

Please sign in to comment.