Add auto retry for package manager based commands

Case RE-996: This makes it so that we automatically retry package
manager commands that fail on the first attempt.  The reason for this
change is that there is a network element involved that can sometimes
fail due to temporary mirror problems and the command can pass after
waiting for a few seconds.

Changelog: Add auto retry for package manager based commands

elevate-cpanel

@@ -9174,7 +9174,22 @@ EOS
 
     sub ssystem_and_die ( $self, @args ) {
         $self->ssystem(@args) or return 0;
-        Carp::croak("command failed. Fix it and run command.");
+
+        my $cmd = join ' ', @args;
+        if ( $args[0] =~ m/yum|dnf|apt|rpm|dpkg/ ) {
+            WARN("Initial attempt to execute '$cmd' failed. Attempting again");
+
+            $self->sleep();
+
+            $self->ssystem(@args) or return 0;
+        }
+
+        Carp::croak("‘$cmd’ failed. Review and fix the error, then try again with ‘/scripts/elevate-cpanel --continue’");
+    }
+
+    sub sleep ($self) {
+        sleep 15;
+        return;
     }
 
     sub _ssystem ( $command, %opts ) {

lib/Elevate/Roles/Run.pm

@@ -67,7 +67,25 @@ sub ssystem ( $, @args ) {
 
 sub ssystem_and_die ( $self, @args ) {
     $self->ssystem(@args) or return 0;
-    Carp::croak("command failed. Fix it and run command.");
+
+    # Allow for one retry when running package manager commands
+    my $cmd = join ' ', @args;
+    if ( $args[0] =~ m/yum|dnf|apt|rpm|dpkg/ ) {
+        WARN("Initial attempt to execute '$cmd' failed. Attempting again");
+
+        # Give it a cool off period before retrying
+        $self->sleep();
+
+        $self->ssystem(@args) or return 0;
+    }
+
+    Carp::croak("‘$cmd’ failed. Review and fix the error, then try again with ‘/scripts/elevate-cpanel --continue’");
+}
+
+# This is done this way for mocking
+sub sleep ($self) {
+    sleep 15;
+    return;
 }
 
 sub _ssystem ( $command, %opts ) {

t/ssystem.t

@@ -96,5 +96,36 @@ is $out, {
   q[ssystem_capture_output( echo -e 'a\nb\nc' )]
   or diag explain $out;
 
+clear_messages_seen();
+
+ok lives { cpev->ssystem_and_die(qw{/bin/true}); }, 'Lives when the command is successful';
+message_seen( INFO => 'Running: /bin/true' );
+message_seen( INFO => '' );
+message_seen( INFO => '' );
+no_messages_seen();
+
+ok dies { cpev->ssystem_and_die(qw{/bin/false}); }, 'Dies when the command fails';
+message_seen( INFO => 'Running: /bin/false' );
+message_seen( INFO => '' );
+message_seen( INFO => '' );
+no_messages_seen();
+
+my $mock_ssystem = Test::MockModule->new('Elevate::Roles::Run');
+$mock_ssystem->redefine(
+    _ssystem => sub { return 0; },
+);
+
+ok lives { cpev->ssystem_and_die(qw{/usr/bin/yum -y install foo}); }, 'Lives when yum is successful';
+no_messages_seen();
+
+$mock_ssystem->redefine(
+    _ssystem => sub { return 42; },
+    sleep    => 1,
+);
+
+ok dies { cpev->ssystem_and_die(qw{/usr/bin/yum -y install foo}); }, 'Lives when yum is successful';
+message_seen( WARN => "Initial attempt to execute '/usr/bin/yum -y install foo' failed. Attempting again" );
+no_messages_seen();
+
 done_testing();
 exit;