diff --git a/isotovideo b/isotovideo index 8a909048666..f92187fa0da 100755 --- a/isotovideo +++ b/isotovideo @@ -31,23 +31,64 @@ ANSI_COLORS_DISABLED or NO_COLOR can be set to any value to disable colors. Show the current program version and test API version +=item B<-e, -?, --exit-status-from-test-results> + +isotovideo will exit with status 1 when a test module fails. + =item B<-h, -?, --help> Show this help. -=head1 TEST PARAMETER +=back + +=head1 TEST PARAMETERS All additional command line arguments specified in the C format are parsed as test parameters which take precedence over the settings in the vars.json file. Lower case key names are transformed into upper case automatically for convenience. +=head1 EXIT CODES + +=over 4 + +=item B<0 - SUCCESS> + +isotovideo successfully executed a complete run. In the case of C<--exit-status-from-test-results> flag additionally all test modules passed or softfailed. + +=item B<1 - ERROR> + +An error ocurred during the test execution related to a test backend failure. + +=item B<100 - NO TEST MODULES SCHEDULED> + +No test module was scheduled. + +This exit code can only be reported when invoked with the +C<--exit-status-from-test-results> flag. + +=item B<101 - TEST MODULES FAILED> + +At least one test module did not end with result "ok" or "softfail". + +This exit code can only be reported when invoked with the +C<--exit-status-from-test-results> flag. + +=back + =cut use Mojo::Base -strict, -signatures; use autodie ':all'; no autodie 'kill'; +use constant { + EXIT_STATUS_OK => 0, + EXIT_STATUS_ERR => 1, + EXIT_STATUS_ERR_NO_TESTS => 100, + EXIT_STATUS_ERR_FROM_TEST_RESULTS => 101, +}; + # Avoid "Subroutine JSON::PP::Boolean::(0+ redefined" warnings # Details: https://progress.opensuse.org/issues/90371 use JSON::PP; @@ -72,14 +113,15 @@ Getopt::Long::Configure("no_ignore_case"); use OpenQA::Isotovideo::Interface; use OpenQA::Isotovideo::Runner; use OpenQA::Isotovideo::Utils qw(git_rev_parse spawn_debuggers handle_generated_assets); +use Mojo::File qw(path); my %options; # global exit status -my $return_code = 1; +my $RETURN_CODE = EXIT_STATUS_ERR; sub usage ($r) { - $return_code = $r; + $RETURN_CODE = $r; eval { require Pod::Usage; Pod::Usage::pod2usage($r) }; die "cannot display help, install perl(Pod::Usage)\n" if $@; } @@ -89,13 +131,30 @@ sub _get_version_string () { return "Current version is $thisversion [interface v$OpenQA::Isotovideo::Interface::version]"; } +sub _exit_code_from_test_results () { + my @results = glob(path(bmwqemu::result_dir(), "result-*.json")); + return EXIT_STATUS_ERR_NO_TESTS if @results == 0; + + my $did_fail = 0; + for my $result_file_path (@results) { + my $result_file = path($result_file_path); + my $test_result = decode_json($result_file->slurp)->{result}; + diag sprintf("Test result [%s] %s", $result_file->to_string, $test_result); + # If a failure (anything different from ok & softfail) is found, keep it. + next if $did_fail; + + $did_fail = $test_result !~ m/^(ok|softfail)$/; + } + return $did_fail ? EXIT_STATUS_ERR_FROM_TEST_RESULTS : EXIT_STATUS_OK; +} + sub version () { print _get_version_string() . "\n"; - $return_code = 0; + $RETURN_CODE = EXIT_STATUS_OK; exit 0; } -GetOptions(\%options, 'debug|d', 'workdir=s', 'color=s', 'help|h|?', 'version|v') or usage(1); +GetOptions(\%options, 'debug|d', 'workdir=s', 'color=s', 'help|h|?', 'version|v', 'exit-status-from-test-results|e') or usage(1); usage(0) if $options{help}; version() if $options{version}; @@ -137,28 +196,32 @@ eval { $runner->handle_commands; - $return_code = 0; + $RETURN_CODE = EXIT_STATUS_OK; # enter the main loop: process messages from autotest, command server and backend $runner->run; + if ($options{'exit-status-from-test-results'}) { + $RETURN_CODE = _exit_code_from_test_results(); + } + # terminate/kill the command server and let it inform its websocket clients before $runner->stop_commands('test execution ended'); if ($runner->testfd) { # unusual shutdown - $return_code = 1; # uncoverable statement + $RETURN_CODE = EXIT_STATUS_ERR; # uncoverable statement CORE::close $runner->testfd; # uncoverable statement $runner->stop_autotest(); # uncoverable statement } - diag 'isotovideo ' . ($return_code ? 'failed' : 'done'); - my $clean_shutdown = $runner->handle_shutdown(\$return_code); + diag 'isotovideo ' . ($RETURN_CODE ? 'failed' : 'done'); + my $clean_shutdown = $runner->handle_shutdown(\$RETURN_CODE); # read calculated variables from backend and tests bmwqemu::load_vars(); - $return_code = handle_generated_assets($runner->command_handler, $clean_shutdown) unless $return_code; + $RETURN_CODE = handle_generated_assets($runner->command_handler, $clean_shutdown) unless $RETURN_CODE; }; if (my $error = $@) { log::fctwarn $error, 'main'; @@ -171,8 +234,8 @@ END { $runner and $runner->stop_autotest(); # in case of early exit, e.g. help display - $return_code //= 0; + $RETURN_CODE //= 0; - print "$$: EXIT $return_code\n"; - $? = $return_code; + print "$$: EXIT $RETURN_CODE\n"; + $? = $RETURN_CODE; } diff --git a/t/14-isotovideo.t b/t/14-isotovideo.t index 35f3d706302..25b9210ccc8 100755 --- a/t/14-isotovideo.t +++ b/t/14-isotovideo.t @@ -7,8 +7,10 @@ use Test::Warnings ':report_warnings'; use Test::MockModule; use FindBin '$Bin'; use lib "$Bin/../external/os-autoinst-common/lib"; -use OpenQA::Test::TimeLimit '20'; use autodie ':all'; + +use OpenQA::Test::TimeLimit '20'; + use IPC::System::Simple qw(system); use Test::Output qw(combined_like combined_from stderr_from); use File::Basename; @@ -238,7 +240,47 @@ subtest 'productdir variable relative/absolute' => sub { unlike $log, qr/assert_screen_fail_test/, 'assert screen test not scheduled'; }; +subtest 'exit status from test results' => sub { + # dummy isotovideo invocations + chdir($pool_dir); + path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE; + path('vars.json')->remove if -e 'vars.json'; + path('serial0')->touch; + + subtest 'no test scheduled' => sub { + path('testresults/')->remove_tree; + my $log = combined_from { isotovideo( + default_opts => '--exit-status-from-test-results backend=null', + opts => "casedir=$data_dir/empty_test_distribution", exit_code => 100) }; + }; + + subtest 'tests scheduled' => sub { + path('testresults/')->remove_tree; + my $module_basename = 'failing_module'; + my $module = "tests/$module_basename"; + my $log = combined_from { isotovideo( + default_opts => '--exit-status-from-test-results backend=null', + opts => "casedir=$data_dir/tests schedule=$module", exit_code => 101) }; + + like $log, qr/scheduling $module_basename $module\.pm/, 'module scheduled'; + like $log, qr/Test result \[testresults\/result\-$module_basename\.json\] fail/, 'failed test module report'; + }; + + subtest 'softfailed module' => sub { + path('testresults/')->remove_tree; + my $module_basename = 'softfail_module'; + my $module = "tests/$module_basename"; + my $log = combined_from { isotovideo( + default_opts => '--exit-status-from-test-results backend=null', + opts => "casedir=$data_dir/tests schedule=$module", exit_code => 0) }; + + like $log, qr/scheduling $module_basename $module\.pm/, 'module scheduled'; + like $log, qr/Test result \[testresults\/result\-$module_basename\.json\] softfail/, 'soft failed test module report'; + }; +}; + subtest 'upload assets on demand even in failed jobs' => sub { + # qemu isotovideo invocation chdir($pool_dir); path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE; path('vars.json')->remove if -e 'vars.json'; @@ -252,6 +294,7 @@ subtest 'upload assets on demand even in failed jobs' => sub { }; subtest 'load test success when casedir and productdir are relative path' => sub { + # qemu isotovideo invocation chdir($pool_dir); path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE; path('vars.json')->remove if -e 'vars.json'; @@ -356,6 +399,7 @@ subtest 'publish assets' => sub { done_testing(); END { + unlink "./serial0" if -e "./serial0"; rmtree "$Bin/data/tests/product"; rmtree "$data_dir/wheels_dir/writer"; rmtree "$pool_dir/writer"; diff --git a/t/data/empty_test_distribution/main.pm b/t/data/empty_test_distribution/main.pm new file mode 100644 index 00000000000..a92c4d6a5b7 --- /dev/null +++ b/t/data/empty_test_distribution/main.pm @@ -0,0 +1,10 @@ +# Copyright SUSE LLC +# SPDX-License-Identifier: GPL-2.0-or-later + +use Mojo::Base -strict; +use testapi; +use autotest; + +# intentionally not loading anything... + +1; diff --git a/t/data/empty_test_distribution/needles/.gitkeep b/t/data/empty_test_distribution/needles/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/t/data/tests/tests/softfail_module.pm b/t/data/tests/tests/softfail_module.pm new file mode 100644 index 00000000000..2eb6e3525da --- /dev/null +++ b/t/data/tests/tests/softfail_module.pm @@ -0,0 +1,11 @@ +# Copyright SUSE LLC +# SPDX-License-Identifier: GPL-2.0-or-later + +use Mojo::Base 'basetest', -signatures; +use testapi; + +sub run ($self) { + record_soft_failure('failing me softly with this song'); +} + +1;