Merge pull request os-autoinst#20169 from alvarocarvajald/ssh_crash_f…

…ailure

Move Public Cloud ssh opts to CLI in HanaSR tests

lib/publiccloud/instance.pm

@@ -242,7 +242,11 @@ sub scp {
     $from =~ s/^remote:/$url/;
     $to =~ s/^remote:/$url/;
 
-    my $ssh_cmd = sprintf('scp %s "%s" "%s"', $self->ssh_opts, $from, $to);
+    # Sanitize ssh_opts by removing -E options which are not accepted by 'scp'
+    my $ssh_opts = $self->ssh_opts;
+    $ssh_opts =~ s/\-E\s[^\s]+//g;
+
+    my $ssh_cmd = sprintf('scp %s "%s" "%s"', $ssh_opts, $from, $to);
 
     return script_run($ssh_cmd, %args);
 }

lib/sles4sap_publiccloud.pm

@@ -22,7 +22,8 @@ use List::MoreUtils qw(uniq);
 use Carp qw(croak);
 use YAML::PP;
 use testapi;
-use utils 'file_content_replace';
+use utils qw(file_content_replace);
+use serial_terminal qw(serial_term_prompt);
 use version_utils qw(check_version);
 use hacluster;
 use qesapdeployment;
@@ -369,13 +370,14 @@ sub stop_hana {
         # Crash needs to be executed as root and wait for host reboot
         $self->{my_instance}->wait_for_ssh(timeout => $timeout);
         $self->{my_instance}->run_ssh_command(cmd => "sudo su -c sync", timeout => "0", %args);
-        # Close SSH mux file before Crash test
-        $self->{my_instance}->run_ssh_command(cmd => " ", timeout => "0", ssh_opts => "-O exit");
+        # Try only extending ssh_opts
+        my $ssh_opts = $self->{my_instance}->ssh_opts . ' -o ServerAliveInterval=2';
         $self->{my_instance}->run_ssh_command(cmd => 'sudo su -c "' . $cmd . '"',
             timeout => "0",
-            # Try only extending ssh_opts
-            ssh_opts => "-o ServerAliveInterval=2 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR",
+            ssh_opts => $ssh_opts,
             %args);
+        # Send a Ctrl-C to unblock the terminal session if no prompt is seen in 30 seconds
+        type_string('', terminate_with => 'ETX') unless (wait_serial(serial_term_prompt()));
         # It is better to wait till ssh disappear
         record_info("Wait ssh disappear start");
         my $out = $self->{my_instance}->wait_for_ssh(timeout => 60, wait_stop => 1);

lib/sles4sap_publiccloud_basetest.pm

@@ -12,12 +12,44 @@ use Mojo::Base 'publiccloud::basetest';
 use strict;
 use warnings FATAL => 'all';
 use Exporter 'import';
+use Carp qw(croak);
 use testapi;
 use qesapdeployment;
 use sles4sap_publiccloud;
+use publiccloud::utils qw(get_ssh_private_key_path);
 
 our @EXPORT = qw(cleanup import_context);
 
+=head1 DESCRIPTION
+
+    Basetest class for SLES for SAP Applications tests in Public Cloud.
+
+=head2 cleanup
+
+    $self->cleanup(%args)
+
+Cleanup method intended to be called at the end of tests or in C<post_fail_hook>.
+Mostly a wrapper around C<sles4sap_publiccloud::sles4sap_cleanup> which will:
+
+=over
+
+=item *
+
+Remove network peerings
+
+=item *
+
+Run ansible de-registration playbooks
+
+=item *
+
+Run C<terraform destroy>
+
+=back
+
+Unless any of these has been executed previously.
+
+=cut
 
 sub cleanup {
     my ($self, $args) = @_;
@@ -42,6 +74,14 @@ sub cleanup {
         && ($self->{result} ne 'fail'));
 }
 
+=head2 import_context
+
+    $self->import_context(%run_args)
+
+Import into C<$self> the class instances context passed via C<%run_args>, and
+record the information in the test results.
+
+=cut
 
 sub import_context {
     my ($self, $run_args) = @_;
@@ -56,6 +96,57 @@ sub import_context {
     );
 }
 
+=head2 set_cli_ssh_opts
+
+    $self->set_cli_ssh_opts();
+    $self->set_cli_ssh_opts('');
+    $self->set_cli_ssh_opts("-4 -o LogLevel=ERROR -E $logfile");
+
+Set command line SSH options in the instance stored in C<$self-E<gt>{my_instance}>. It takes
+as an argument a string with the options in a manner that would be understood by B<ssh>, and
+if no argument is provided, uses the following defaults:
+
+=over
+
+=item *
+
+C<-E /var/tmp/ssh_sut.log>: save logging to B</var/tmp/ssh_sut.log>.
+
+=item *
+
+C<-F none>: do not use SSH configuration files.
+
+=item *
+
+C<-o LogLevel=DEBUG3>: set log level to B<DEBUG3>.
+
+=item *
+
+C<-o PasswordAutentication=no>: do not allow authentication via password.
+
+=item *
+
+C<-i 'get_ssh_private_key_path()'>: use the generated private SSH key
+
+=back
+
+B<Note>: if the method receives an empty string, no SSH options will be set.
+
+=cut
+
+sub set_cli_ssh_opts {
+    my ($self, $ssh_opts) = @_;
+    croak("Expected \$self->{my_instance} is not defined. Check module Description for details")
+      unless $self->{my_instance};
+    $ssh_opts //= join(' ',
+        '-E', '/var/tmp/ssh_sut.log',
+        '-F', 'none',
+        '-o', 'LogLevel=DEBUG3',
+        '-o', 'PasswordAuthentication=no',
+        '-i', "'" . get_ssh_private_key_path() . "'");
+    $self->{my_instance}->ssh_opts($ssh_opts);
+}
+
 sub post_fail_hook {
     my ($self) = @_;
     if (get_var('QESAP_NO_CLEANUP_ON_FAILURE')) {

t/12_sles4sap_publicccloud.t

@@ -161,8 +161,11 @@ subtest "[stop_hana]" => sub {
 subtest "[stop_hana] crash" => sub {
     my $sles4sap_publiccloud = Test::MockModule->new('sles4sap_publiccloud', no_auto => 1);
     $sles4sap_publiccloud->redefine(record_info => sub { note(join(' ', 'RECORD_INFO -->', @_)); });
+    $sles4sap_publiccloud->redefine(type_string => sub { note(join(' ', 'TYPED -->', @_)); });
     $sles4sap_publiccloud->redefine(wait_for_sync => sub { return; });
     $sles4sap_publiccloud->redefine(wait_hana_node_up => sub { return; });
+    $sles4sap_publiccloud->redefine(serial_term_prompt => sub { return '# '; });
+    $sles4sap_publiccloud->redefine(wait_serial => sub { return; });
 
     my $self = sles4sap_publiccloud->new();
     my $mock_pc = Test::MockObject->new();
@@ -172,6 +175,7 @@ subtest "[stop_hana] crash" => sub {
             my ($self, %args) = @_;
             push @calls, $args{cmd};
             return 'BABUUUUUUUUM' });
+    $mock_pc->mock('ssh_opts', sub { return '-i .ssh/id_rsa'; });
     $self->{my_instance} = $mock_pc;
 
     $self->stop_hana(method => 'crash');

tests/sles4sap/publiccloud/qesap_terraform.pm

@@ -210,6 +210,7 @@ sub run {
     foreach my $instance (@$instances) {
         record_info 'Instance', join(' ', 'IP: ', $instance->public_ip, 'Name: ', $instance->instance_id);
         $self->{my_instance} = $instance;
+        $self->set_cli_ssh_opts unless (get_var('MR_TEST', 0));    # Set CLI SSH opts in HanaSR test, not in saptune/mr_test tests
         my $expected_hostname = $instance->{instance_id};
         $instance->wait_for_ssh();
         # Does not fail for some reason.