Report test module failures via exit codes

- Introduced a `--exit-status-from-test-results` / `-e` flag to `isotovideo`. It'll inspect the test results after a full run and compute the exit code from them. If: - no test results exist: exit code = 100 - failed test results results exist: exit code = 101 (failed result => anything different than ok or softfail) else - exit code = 0 - Documented the flags in the perl pod section. - Unit tested with an empty distribution, a fail module distribution and a soft failure. - X-mas gift: Uppercase `$return_code` to follow perl practices. Co-authored-by: Oliver Kurz <[email protected]> Co-authored-by: Martchus <[email protected]>
josegomezr · Jan 9, 2024 · 5eb5dfb · 5eb5dfb
1 parent bd10c5a
commit 5eb5dfb
Show file tree

Hide file tree

Showing 5 changed files with 142 additions and 14 deletions.
diff --git a/isotovideo b/isotovideo
@@ -31,23 +31,64 @@ ANSI_COLORS_DISABLED or NO_COLOR can be set to any value to disable colors.
 
 Show the current program version and test API version
 
+=item B<-e, -?, --exit-status-from-test-results>
+
+isotovideo will exit with status 1 when a test module fails.
+
 =item B<-h, -?, --help>
 
 Show this help.
 
-=head1 TEST PARAMETER
+=back
+
+=head1 TEST PARAMETERS
 
 All additional command line arguments specified in the C<key=value> format are
 parsed as test parameters which take precedence over the settings in the
 vars.json file. Lower case key names are transformed into upper case
 automatically for convenience.
 
+=head1 EXIT CODES
+
+=over 4
+
+=item B<0 - SUCCESS>
+
+isotovideo successfully executed a complete run. In the case of C<--exit-status-from-test-results> flag additionally all test modules passed or softfailed.
+
+=item B<1 - ERROR>
+
+An error ocurred during the test execution related to a test backend failure.
+
+=item B<100 - NO TEST MODULES SCHEDULED>
+
+No test module was scheduled.
+
+This exit code can only be reported when invoked with the
+C<--exit-status-from-test-results> flag.
+
+=item B<101 - TEST MODULES FAILED>
+
+At least one test module did not end with result "ok" or "softfail".
+
+This exit code can only be reported when invoked with the
+C<--exit-status-from-test-results> flag.
+
+=back
+
 =cut
 
 use Mojo::Base -strict, -signatures;
 use autodie ':all';
 no autodie 'kill';
 
+use constant {
+    EXIT_STATUS_OK => 0,
+    EXIT_STATUS_ERR => 1,
+    EXIT_STATUS_ERR_NO_TESTS => 100,
+    EXIT_STATUS_ERR_FROM_TEST_RESULTS => 101,
+};
+
 # Avoid "Subroutine JSON::PP::Boolean::(0+ redefined" warnings
 # Details: https://progress.opensuse.org/issues/90371
 use JSON::PP;
@@ -72,14 +113,15 @@ Getopt::Long::Configure("no_ignore_case");
 use OpenQA::Isotovideo::Interface;
 use OpenQA::Isotovideo::Runner;
 use OpenQA::Isotovideo::Utils qw(git_rev_parse spawn_debuggers handle_generated_assets);
+use Mojo::File qw(path);
 
 my %options;
 
 # global exit status
-my $return_code = 1;
+my $RETURN_CODE = EXIT_STATUS_ERR;
 
 sub usage ($r) {
-    $return_code = $r;
+    $RETURN_CODE = $r;
     eval { require Pod::Usage; Pod::Usage::pod2usage($r) };
     die "cannot display help, install perl(Pod::Usage)\n" if $@;
 }
@@ -89,13 +131,30 @@ sub _get_version_string () {
     return "Current version is $thisversion [interface v$OpenQA::Isotovideo::Interface::version]";
 }
 
+sub _exit_code_from_test_results () {
+    my @results = glob(path(bmwqemu::result_dir(), "result-*.json"));
+    return EXIT_STATUS_ERR_NO_TESTS if @results == 0;
+
+    my $did_fail = 0;
+    for my $result_file_path (@results) {
+        my $result_file = path($result_file_path);
+        my $test_result = decode_json($result_file->slurp)->{result};
+        diag sprintf("Test result [%s] %s", $result_file->to_string, $test_result);
+        # If a failure (anything different from ok & softfail) is found, keep it.
+        next if $did_fail;
+
+        $did_fail = $test_result !~ m/^(ok|softfail)$/;
+    }
+    return $did_fail ? EXIT_STATUS_ERR_FROM_TEST_RESULTS : EXIT_STATUS_OK;
+}
+
 sub version () {
     print _get_version_string() . "\n";
-    $return_code = 0;
+    $RETURN_CODE = EXIT_STATUS_OK;
     exit 0;
 }
 
-GetOptions(\%options, 'debug|d', 'workdir=s', 'color=s', 'help|h|?', 'version|v') or usage(1);
+GetOptions(\%options, 'debug|d', 'workdir=s', 'color=s', 'help|h|?', 'version|v', 'exit-status-from-test-results|e') or usage(1);
 usage(0) if $options{help};
 version() if $options{version};
 
@@ -137,28 +196,32 @@ eval {
 
     $runner->handle_commands;
 
-    $return_code = 0;
+    $RETURN_CODE = EXIT_STATUS_OK;
 
     # enter the main loop: process messages from autotest, command server and backend
     $runner->run;
 
+    if ($options{'exit-status-from-test-results'}) {
+        $RETURN_CODE = _exit_code_from_test_results();
+    }
+
     # terminate/kill the command server and let it inform its websocket clients before
     $runner->stop_commands('test execution ended');
 
     if ($runner->testfd) {
         # unusual shutdown
-        $return_code = 1;    # uncoverable statement
+        $RETURN_CODE = EXIT_STATUS_ERR;    # uncoverable statement
         CORE::close $runner->testfd;    # uncoverable statement
         $runner->stop_autotest();    # uncoverable statement
     }
 
-    diag 'isotovideo ' . ($return_code ? 'failed' : 'done');
-    my $clean_shutdown = $runner->handle_shutdown(\$return_code);
+    diag 'isotovideo ' . ($RETURN_CODE ? 'failed' : 'done');
+    my $clean_shutdown = $runner->handle_shutdown(\$RETURN_CODE);
 
     # read calculated variables from backend and tests
     bmwqemu::load_vars();
 
-    $return_code = handle_generated_assets($runner->command_handler, $clean_shutdown) unless $return_code;
+    $RETURN_CODE = handle_generated_assets($runner->command_handler, $clean_shutdown) unless $RETURN_CODE;
 };
 if (my $error = $@) {
     log::fctwarn $error, 'main';
@@ -171,8 +234,8 @@ END {
     $runner and $runner->stop_autotest();
 
     # in case of early exit, e.g. help display
-    $return_code //= 0;
+    $RETURN_CODE //= 0;
 
-    print "$$: EXIT $return_code\n";
-    $? = $return_code;
+    print "$$: EXIT $RETURN_CODE\n";
+    $? = $RETURN_CODE;
 }
diff --git a/t/14-isotovideo.t b/t/14-isotovideo.t
@@ -7,8 +7,10 @@ use Test::Warnings ':report_warnings';
 use Test::MockModule;
 use FindBin '$Bin';
 use lib "$Bin/../external/os-autoinst-common/lib";
-use OpenQA::Test::TimeLimit '20';
 use autodie ':all';
+
+use OpenQA::Test::TimeLimit '20';
+
 use IPC::System::Simple qw(system);
 use Test::Output qw(combined_like combined_from stderr_from);
 use File::Basename;
@@ -238,7 +240,47 @@ subtest 'productdir variable relative/absolute' => sub {
     unlike $log, qr/assert_screen_fail_test/, 'assert screen test not scheduled';
 };
 
+subtest 'exit status from test results' => sub {
+    # dummy isotovideo invocations
+    chdir($pool_dir);
+    path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE;
+    path('vars.json')->remove if -e 'vars.json';
+    path('serial0')->touch;
+
+    subtest 'no test scheduled' => sub {
+        path('testresults/')->remove_tree;
+        my $log = combined_from { isotovideo(
+                default_opts => '--exit-status-from-test-results backend=null',
+                opts => "casedir=$data_dir/empty_test_distribution", exit_code => 100) };
+    };
+
+    subtest 'tests scheduled' => sub {
+        path('testresults/')->remove_tree;
+        my $module_basename = 'failing_module';
+        my $module = "tests/$module_basename";
+        my $log = combined_from { isotovideo(
+                default_opts => '--exit-status-from-test-results backend=null',
+                opts => "casedir=$data_dir/tests schedule=$module", exit_code => 101) };
+
+        like $log, qr/scheduling $module_basename $module\.pm/, 'module scheduled';
+        like $log, qr/Test result \[testresults\/result\-$module_basename\.json\] fail/, 'failed test module report';
+    };
+
+    subtest 'softfailed module' => sub {
+        path('testresults/')->remove_tree;
+        my $module_basename = 'softfail_module';
+        my $module = "tests/$module_basename";
+        my $log = combined_from { isotovideo(
+                default_opts => '--exit-status-from-test-results backend=null',
+                opts => "casedir=$data_dir/tests schedule=$module", exit_code => 0) };
+
+        like $log, qr/scheduling $module_basename $module\.pm/, 'module scheduled';
+        like $log, qr/Test result \[testresults\/result\-$module_basename\.json\] softfail/, 'soft failed test module report';
+    };
+};
+
 subtest 'upload assets on demand even in failed jobs' => sub {
+    # qemu isotovideo invocation
     chdir($pool_dir);
     path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE;
     path('vars.json')->remove if -e 'vars.json';
@@ -252,6 +294,7 @@ subtest 'upload assets on demand even in failed jobs' => sub {
 };
 
 subtest 'load test success when casedir and productdir are relative path' => sub {
+    # qemu isotovideo invocation
     chdir($pool_dir);
     path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE;
     path('vars.json')->remove if -e 'vars.json';
@@ -356,6 +399,7 @@ subtest 'publish assets' => sub {
 done_testing();
 
 END {
+    unlink "./serial0" if -e "./serial0";
     rmtree "$Bin/data/tests/product";
     rmtree "$data_dir/wheels_dir/writer";
     rmtree "$pool_dir/writer";

diff --git a/t/data/empty_test_distribution/main.pm b/t/data/empty_test_distribution/main.pm
@@ -0,0 +1,10 @@
+# Copyright SUSE LLC
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+use Mojo::Base -strict;
+use testapi;
+use autotest;
+
+# intentionally not loading anything...
+
+1;
diff --git a/t/data/empty_test_distribution/needles/.gitkeep b/t/data/empty_test_distribution/needles/.gitkeep
diff --git a/t/data/tests/tests/softfail_module.pm b/t/data/tests/tests/softfail_module.pm
@@ -0,0 +1,11 @@
+# Copyright SUSE LLC
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+use Mojo::Base 'basetest', -signatures;
+use testapi;
+
+sub run ($self) {
+    record_soft_failure('failing me softly with this song');
+}
+
+1;