Version 1.1.1

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.1.0
+current_version = 1.1.1
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)

CHANGELOG.md

@@ -4,7 +4,11 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [1.1.1] - 2024-10-02
+
+### Changed
+- **Rsync** - Properly triggers retry sequence
+- **Rsync** - Gives a return code now
 
 ### Fixed
 - **Create** - Fix GPU AMI not being selected.

src/forge/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.1.0"
+__version__ = "1.1.1"
 
 # Default values for forge's essential arguments
 DEFAULT_ARG_VALS = {

src/forge/engine.py

@@ -54,7 +54,7 @@ def engine(config):
 
     try:
         create(config)
-        rsync(config)
+        status = rsync(config)
         status = run(config)
     except ExitHandlerException:
         # Check for spot instances and retries

src/forge/main.py

@@ -83,7 +83,7 @@ def execute(config):
     elif job == 'destroy':
         destroy(config)
     elif job == 'rsync':
-        rsync(config)
+        status = rsync(config)
     elif job == 'run':
         status = run(config)
     elif job == 'engine':

src/forge/rsync.py

@@ -5,6 +5,8 @@
 import sys
 
 from . import DEFAULT_ARG_VALS, REQUIRED_ARGS
+from .destroy import destroy
+from .exceptions import ExitHandlerException
 from .parser import add_basic_args, add_general_args, add_env_args, add_action_args
 from .common import ec2_ip, key_file, get_ip, get_nlist, exit_callback
 
@@ -39,8 +41,16 @@ def rsync(config):
     ----------
     config : dict
         Forge configuration data
+
+    Returns
+    -------
+    int
+        The status of the rsync commands
     """
 
+    destroy_flag = config.get('destroy_after_failure')
+    rval = 0
+
     def _rsync(config, ip):
         """performs the rsync to a given ip
 
@@ -67,17 +77,17 @@ def _rsync(config, ip):
                 sys.exit(1)
 
             cmd = 'rsync -rave "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
-            cmd +=f' -i {pem_path}" {rsync_loc} root@{ip}:/root/'
+            cmd += f' -i {pem_path}" {rsync_loc} root@{ip}:/root/'
 
             try:
                 output = subprocess.check_output(
                     cmd, stderr=subprocess.STDOUT, shell=True, universal_newlines=True
                 )
+                logger.info('Rsync successful:\n%s', output)
+                return 0
             except subprocess.CalledProcessError as exc:
                 logger.error('Rsync failed:\n%s', exc.output)
-                exit_callback(config)
-            else:
-                logger.info('Rsync successful:\n%s', output)
+                return exc.returncode
 
     n_list = get_nlist(config)
 
@@ -93,6 +103,14 @@ def _rsync(config, ip):
 
             for ip, _ in targets:
                 logger.info('Rsync destination is %s', ip)
-                _rsync(config, ip)
-        except Exception as e:
+                rval = _rsync(config, ip)
+                if rval:
+                    raise ValueError('Rsync command unsuccessful, ending attempts.')
+        except ValueError as e:
             logger.error('Got error %s when trying to rsync.', e)
+            try:
+                exit_callback(config)
+            except ExitHandlerException:
+                raise
+
+    return rval

src/forge/run.py

@@ -123,7 +123,7 @@ def _run(config, ip):
             logger.error('Run command raised error: %s', e)
             try:
                 exit_callback(config)
-            except ExitHandlerException as exc:
+            except ExitHandlerException:
                 raise
             finally:
                 if destroy_flag:

tests/test_rsync.py

@@ -167,6 +167,7 @@ def test_rsync_fail(mock_ec2_ip, mock_get_ip, mock_key_file, mock_sub_chk,
         'region': 'us-east-1',
         'aws_profile': 'dev',
         'rsync_path': rsync_path,
+        'job': 'rsync',
     }
     expected_cmd = 'rsync -rave "ssh -o UserKnownHostsFile=/dev/null -o'
     expected_cmd += f' StrictHostKeyChecking=no -i {key_path}" {rsync_path}/* root@{ip}:/root/'
@@ -175,7 +176,7 @@ def test_rsync_fail(mock_ec2_ip, mock_get_ip, mock_key_file, mock_sub_chk,
         returncode=123, cmd=expected_cmd
     )
 
-    rsync.rsync(config)
+    assert rsync.rsync(config) == 123
 
     mock_ec2_ip.assert_called_once_with(
         f"{config['name']}-spot-single-{config['date']}", config