Merge pull request #20598 from lilyeyes/fix-sdaf-post-fail-hook-not-e…

…xecuted-issue

Fix SDAF Post fail hook not executed when timeout

lib/sles4sap/console_redirection.pm

@@ -175,11 +175,6 @@ from serial console. VM ID is collected by opening 'log-console' which is not re
 sub check_serial_redirection {
     # Do not select serial console if it is already done. This avoids log pollution and speeds up process.
     if (is_serial_terminal()) {
-        # Teminate the process when scripts timed out
-        unless (wait_serial($testapi::distri->{serial_term_prompt})) {
-            type_string('', terminate_with => 'ETX');
-            type_string("\n");
-        }
         select_serial_terminal();
         set_serial_term_prompt();
     }

lib/sles4sap/sap_deployment_automation_framework/basetest.pm

@@ -13,12 +13,13 @@ use strict;
 use warnings;
 use testapi;
 use Exporter qw(import);
-use sles4sap::sap_deployment_automation_framework::deployment qw(sdaf_cleanup az_login load_os_env_variables);
+use sles4sap::sap_deployment_automation_framework::deployment qw(sdaf_cleanup az_login load_os_env_variables $output_log_file);
 use sles4sap::sap_deployment_automation_framework::deployment_connector
   qw(find_deployer_resources destroy_deployer_vm get_deployer_vm_name find_deployment_id get_deployer_ip destroy_orphaned_resources);
 use sles4sap::console_redirection;
 
-our @EXPORT = qw(full_cleanup);
+our @EXPORT = qw(full_cleanup $serial_regexp_playbook);
+our $serial_regexp_playbook = 0;
 
 =head1 SYNOPSIS
 
@@ -68,9 +69,10 @@ sub full_cleanup {
         # Do not fail even if connection is not successful
         $redirection_works = connect_target_to_serial(fail_ok => '1');
     }
-    my $sut_cleanup_message = $redirection_works ?
-      'Console redirection to Deployer VM does not seem to work. Destroying SUT infrastructure is not possible.' :
-      'Console redirection works, proceeding with SUT cleanup';
+    my $sut_cleanup_message
+      = $redirection_works
+      ? 'Console redirection to Deployer VM does not seem to work. Destroying SUT infrastructure is not possible.'
+      : 'Console redirection works, proceeding with SUT cleanup';
     record_info('SUT cleanup', $sut_cleanup_message);
 
     # Trigger SDAF remover script to destroy 'workload zone' and 'sap systems' resources
@@ -95,6 +97,32 @@ sub full_cleanup {
 
 sub post_fail_hook {
     record_info('Post fail', 'Executing post fail hook');
+    if (testapi::is_serial_terminal()) {
+        # In case playbook/script times out, it will keep occupying the command line,
+        # therefore we need to press Ctrl+c to terminate the process.
+        # Note:
+        #   For playbook please do not use '> ' (or $testapi::distri->{serial_term_prompt})
+        #   as there are lots of '> ' in the output of playbook.
+        #   For playbook/script 'qr/-\d+-/' is usually the last output from playbook/script execution
+        my $match_re = $testapi::distri->{serial_term_prompt};
+        $match_re = qr/-\d+-/ if ($serial_regexp_playbook);
+        unless (wait_serial($match_re)) {
+            type_string('', terminate_with => 'ETX');
+            # Wait for process returns
+            wait_serial(qr/-\d+-/);
+            type_string("\n");
+            if ($serial_regexp_playbook) {
+                record_info('Terminated playbook process');
+                $serial_regexp_playbook = 0;
+                # Upload playbook log file
+                upload_logs($sles4sap::sap_deployment_automation_framework::deployment::output_log_file);
+            }
+            else {
+                record_info('Terminated other script process');
+            }
+        }
+    }
+
     full_cleanup();
 }
 

lib/sles4sap/sap_deployment_automation_framework/deployment.pm

@@ -32,6 +32,24 @@ use sles4sap::sap_deployment_automation_framework::naming_conventions qw(
   get_workload_vnet_code
 );
 
+our @EXPORT = qw(
+  az_login
+  sdaf_ssh_key_from_keyvault
+  serial_console_diag_banner
+  set_common_sdaf_os_env
+  prepare_sdaf_project
+  set_os_variable
+  get_os_variable
+  sdaf_execute_deployment
+  load_os_env_variables
+  sdaf_cleanup
+  sdaf_execute_playbook
+  ansible_hanasr_show_status
+  $output_log_file
+);
+
+our $output_log_file = '';
+
 =head1 SYNOPSIS
 
 Library with common functions for Microsoft SDAF deployment automation. Documentation can be found on the
@@ -64,22 +82,6 @@ Since SUT VMs have no public IPs, this is also serving as a jump-host to reach t
 =back
 =cut
 
-our @EXPORT = qw(
-  az_login
-  sdaf_ssh_key_from_keyvault
-  serial_console_diag_banner
-  set_common_sdaf_os_env
-  prepare_sdaf_project
-  set_os_variable
-  get_os_variable
-  sdaf_execute_deployment
-  load_os_env_variables
-  sdaf_cleanup
-  sdaf_execute_playbook
-  ansible_hanasr_show_status
-);
-
-
 =head2 log_command_output
 
     log_command_output(command=>$command, log_file=>$log_file);
@@ -460,7 +462,7 @@ sub sdaf_execute_deployment {
 
     record_info('SDAF exe', "Executing '$args{deployment_type}' deployment: $deploy_command");
     my $rc;
-    my $output_log_file = log_dir() . "/deploy_$args{deployment_type}_attempt.txt";
+    $output_log_file = log_dir() . "/deploy_$args{deployment_type}_attempt.txt";
     my $attempt_no = 1;
     while ($attempt_no <= $args{retries}) {
         $output_log_file =~ s/attempt/attempt-$attempt_no/;
@@ -665,7 +667,7 @@ sub sdaf_execute_remover {
         '--auto-approve');
 
     my $rc;
-    my $output_log_file = log_dir() . "/cleanup_$args{deployment_type}_attempt.txt";
+    $output_log_file = log_dir() . "/cleanup_$args{deployment_type}_attempt.txt";
     my $attempt_no = 1;
     # SDAF must be executed from the profile directory, otherwise it will fail
     assert_script_run("cd " . $tfvars_path);
@@ -770,7 +772,7 @@ sub sdaf_execute_playbook {
         '--ssh-common-args="-o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=120"'
     );
 
-    my $output_log_file = log_dir() . "/$args{playbook_filename}" =~ s/.yaml|.yml/.txt/r;
+    $output_log_file = log_dir() . "/$args{playbook_filename}" =~ s/.yaml|.yml/.txt/r;
     my $playbook_file = join('/', deployment_dir(), 'sap-automation', 'deploy', 'ansible', $args{playbook_filename});
     my $playbook_cmd = join(' ', 'ansible-playbook', $playbook_options, $playbook_file);
 
@@ -838,3 +840,5 @@ sub ansible_hanasr_show_status {
     record_info('CRM status', script_output(join(' ', @cmd, '--args="sudo crm status full"', '2> /dev/null')));
     record_info('HANA SR', script_output(join(' ', @cmd, '--args="sudo SAPHanaSR-showAttr"', '2> /dev/null')));
 }
+
+1;

t/19_console_redirection.t

@@ -104,6 +104,7 @@ subtest '[check_serial_redirection]' => sub {
     $redirect->redefine(select_serial_terminal => sub { return; });
     $redirect->redefine(set_serial_term_prompt => sub { return; });
     $redirect->redefine(is_serial_terminal => sub { return; });
+    $redirect->redefine(disconnect_target_from_serial => sub { return; });
     set_var('QEMUPORT', '1988');
 
     $redirect->redefine(script_run => sub { return '0'; });
@@ -112,18 +113,6 @@ subtest '[check_serial_redirection]' => sub {
     $redirect->redefine(script_run => sub { $executed_command = $_[0]; return '1'; });
     is check_serial_redirection(), '1', 'Return 1 if machine IDs do not match';
     is $executed_command, 'grep 7902847fcc554911993686a1d5eca2c8 /etc/machine-id', 'Check executed command';
-
-    # Test wait_serial is true
-    $redirect->redefine(is_serial_terminal => sub { return '1'; });
-    $redirect->redefine(wait_serial => sub { return '1'; });
-    $redirect->redefine(script_run => sub { return '0'; });
-    is check_serial_redirection(), '0', 'Return 0 if machine IDs match';
-
-    # Test wait_serial is false
-    $redirect->redefine(wait_serial => sub { return '0'; });
-    $redirect->redefine(type_string => sub { return; });
-    is check_serial_redirection(), '0', 'Return 0 if machine IDs match';
-
     unset_vars();
 };
 

tests/sles4sap/sap_deployment_automation_framework/deploy_hanasr.pm

@@ -84,7 +84,9 @@ sub run {
                 '--args="sudo zypper in -y fence-agents-azure-arm"');
             assert_script_run(join(' ', @cmd));
         }
+        $sles4sap::sap_deployment_automation_framework::basetest::serial_regexp_playbook = 1;
         sdaf_execute_playbook(%{$playbook_options}, sdaf_config_root_dir => $sdaf_config_root_dir);
+        $sles4sap::sap_deployment_automation_framework::basetest::serial_regexp_playbook = 0;
     }
 
     # Display deployment information