Report test module failures via exit codes

- Introduced a `--exit-status-from-test-results` / `-e` flag to
  `isotovideo`.

  It'll inspect the test results after a full run and compute the exit
  code from them.

  If:
  - no test results exist: exit code = 100
  - failed test results results exist: exit code = 101
    (failed result => anything different than ok)
  else
  - exit code = 0

- Documented the flags in the perl pod section.

- Unit tested with an empty distribution & a fail module distribution.

isotovideo

@@ -31,23 +31,67 @@ ANSI_COLORS_DISABLED or NO_COLOR can be set to any value to disable colors.
 
 Show the current program version and test API version
 
+=item B<-e, -?, --exit-status-from-test-results>
+
+isotovideo will exit with code 1 when a test module fails.
+
 =item B<-h, -?, --help>
 
 Show this help.
 
-=head1 TEST PARAMETER
+=back
+
+=head1 TEST PARAMETERS
 
 All additional command line arguments specified in the C<key=value> format are
 parsed as test parameters which take precedence over the settings in the
 vars.json file. Lower case key names are transformed into upper case
 automatically for convenience.
 
+=head1 EXIT CODES
+
+=over 4
+
+=item B<0 - SUCCESS>
+
+Test ran successfully. Bear in mind that if any of the test modules scheduled
+failed the exit code I<will> be 0.
+
+C<isotovideo> can return non-zero exit codes on test module failures with the
+C<--exit-status-from-test-results> flag.
+
+=item B<0 - ERROR>
+
+An error ocurred during the test execution related to a test backend failure.
+
+=item B<100 - NO TEST MODULES SCHEDULED>
+
+No test module was scheduled.
+
+This exit code is reported when invoked with the C<--exit-status-from-test-results> flag.
+
+
+=item B<101 - TEST MODULES FAILED>
+
+A test module failed.
+
+This exit code is reported when invoked with the C<--exit-status-from-test-results> flag.
+
+=back
+
 =cut
 
 use Mojo::Base -strict, -signatures;
 use autodie ':all';
 no autodie 'kill';
 
+use constant {
+    EXIT_STATUS_OK => 0,
+    EXIT_STATUS_ERR => 1,
+    EXIT_STATUS_ERR_NO_TESTS => 100,
+    EXIT_STATUS_ERR_FROM_TEST_RESULTS => 101,
+};
+
 # Avoid "Subroutine JSON::PP::Boolean::(0+ redefined" warnings
 # Details: https://progress.opensuse.org/issues/90371
 use JSON::PP;
@@ -72,11 +116,12 @@ Getopt::Long::Configure("no_ignore_case");
 use OpenQA::Isotovideo::Interface;
 use OpenQA::Isotovideo::Runner;
 use OpenQA::Isotovideo::Utils qw(git_rev_parse spawn_debuggers handle_generated_assets);
+use Mojo::File qw(path);
 
 my %options;
 
 # global exit status
-my $return_code = 1;
+my $return_code = EXIT_STATUS_ERR;
 
 sub usage ($r) {
     $return_code = $r;
@@ -89,13 +134,33 @@ sub _get_version_string () {
     return "Current version is $thisversion [interface v$OpenQA::Isotovideo::Interface::version]";
 }
 
+sub _exit_code_from_test_results() {
+    my @results = glob(path(bmwqemu::result_dir(), "/result-*.json"));
+
+    return EXIT_STATUS_ERR_NO_TESTS if scalar @results == 0;
+
+    my $did_fail = 0;
+
+    for my $result_file_path (@results) {
+        my $result_file = path(Mojo::Path->new($result_file_path)->canonicalize);
+        my $test_result = decode_json($result_file->slurp)->{result};
+        diag sprintf("Test result [%s] %s", $result_file->to_string, $test_result);
+
+        # If a failure is found, keep the failure
+        $did_fail = $test_result eq 'ok' ? 0 : 1 unless $did_fail;
+    }
+
+    return $did_fail ? EXIT_STATUS_ERR_FROM_TEST_RESULTS : EXIT_STATUS_OK;
+}
+
+
 sub version () {
     print _get_version_string() . "\n";
-    $return_code = 0;
+    $return_code = EXIT_STATUS_OK;
     exit 0;
 }
 
-GetOptions(\%options, 'debug|d', 'workdir=s', 'color=s', 'help|h|?', 'version|v') or usage(1);
+GetOptions(\%options, 'debug|d', 'workdir=s', 'color=s', 'help|h|?', 'version|v', 'exit-status-from-test-results|e') or usage(1);
 usage(0) if $options{help};
 version() if $options{version};
 
@@ -137,17 +202,24 @@ eval {
 
     $runner->handle_commands;
 
-    $return_code = 0;
+    $return_code = EXIT_STATUS_OK;
 
     # enter the main loop: process messages from autotest, command server and backend
     $runner->run;
 
+    if ($options{'exit-status-from-test-results'}) {
+        # Ideally: $return_code = $runner->command_handler->test_completed == 0;
+        #          should do it, but autotest returns success if the last
+        #          module ran successfully. Rendering this useless...
+        $return_code = _exit_code_from_test_results();
+    }
+
     # terminate/kill the command server and let it inform its websocket clients before
     $runner->stop_commands('test execution ended');
 
     if ($runner->testfd) {
         # unusual shutdown
-        $return_code = 1;    # uncoverable statement
+        $return_code = EXIT_STATUS_ERR;    # uncoverable statement
         CORE::close $runner->testfd;    # uncoverable statement
         $runner->stop_autotest();    # uncoverable statement
     }

t/14-isotovideo.t

@@ -238,6 +238,30 @@ subtest 'productdir variable relative/absolute' => sub {
     unlike $log, qr/assert_screen_fail_test/, 'assert screen test not scheduled';
 };
 
+subtest 'exit status from test results: no test scheduled' => sub {
+    chdir($pool_dir);
+    path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE;
+    path('vars.json')->remove if -e 'vars.json';
+    my $module = 'tests/failing_module';
+    my $log = combined_from { isotovideo(
+            default_opts => '--exit-status-from-test-results backend=null',
+            opts => "casedir=$data_dir/empty_test_distribution qemu_no_kvm=1 arch=i386 backend=qemu qemu=i386 novideo=1", exit_code => 100) };
+};
+
+subtest 'exit status from test results: tests scheduled' => sub {
+    chdir($pool_dir);
+    path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE;
+    path('vars.json')->remove if -e 'vars.json';
+    my $module_basename = 'failing_module';
+    my $module = "tests/$module_basename";
+    my $log = combined_from { isotovideo(
+            default_opts => '--exit-status-from-test-results backend=null',
+            opts => "casedir=$data_dir/tests schedule=$module qemu_no_kvm=1 arch=i386 backend=qemu qemu=i386 novideo=1", exit_code => 101) };
+
+    like $log, qr/scheduling failing_module $module\.pm/, 'module scheduled';
+    like $log, qr/Test result \[testresults\/result\-$module_basename\.json\] fail/, 'module test result parsed';
+};
+
 subtest 'upload assets on demand even in failed jobs' => sub {
     chdir($pool_dir);
     path(bmwqemu::STATE_FILE)->remove if -e bmwqemu::STATE_FILE;

t/data/empty_test_distribution/main.pm

@@ -0,0 +1,12 @@
+# Copyright SUSE LLC
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+use strict;
+use warnings;
+
+use testapi;
+use autotest;
+
+# intentionally not loading anything...
+
+1;

t/data/empty_test_distribution/needles/.gitkeep

@@ -0,0 +1,12 @@
+# Copyright SUSE LLC
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+use strict;
+use warnings;
+
+use testapi;
+use autotest;
+
+# intentionally not loading anything...
+
+1;