CFY-7569: Use temporary directory instead of current directory for te…

…mp files

agent_packager/cli.py

@@ -3,16 +3,15 @@
 """Script to run Cloudify's Agent Packager via command line
 
 Usage:
-    cfy-ap [--config=<path> --force --dryrun --no-validation -v --env=<env_path>]
+    cfy-ap [--config=<path> --force --dryrun --no-validation -v]
     cfy-ap --version
 
 Options:
     -h --help                   Show this screen
     -c --config=<path>          Path to config yaml (defaults to config.yaml)
-    -f --force                  Forces deletion and creation of venv and tar file.
+    -f --force                  Forces deletion and creation of tar file.
     -d --dryrun                 Prints out the modules to be installed without actually installing them.
     -n --no-validation          Does not validate that all modules were installed correctly.
-    -e --env=<env_path>         Virtualenv to use (defaults to a temporary directory)
     -v --verbose                verbose level logging
     --version                   Display current version
 """
@@ -48,8 +47,7 @@ def _run(test_options=None):
         force=options.get('--force'),
         dryrun=options.get('--dryrun'),
         no_validate=options.get('--no-validation'),
-        verbose=options.get('--verbose'),
-        virtualenv=options.get('--env')
+        verbose=options.get('--verbose')
         )
 
 def main():

agent_packager/packager.py

@@ -38,6 +38,7 @@
 ]
 
 DEFAULT_CLOUDIFY_AGENT_URL = 'https://github.com/cloudify-cosmo/cloudify-agent/archive/{0}.tar.gz'  # NOQA
+VENV_ROOT = ['cloudify', 'env']
 
 lgr = logger.init()
 verbose_output = False
@@ -77,30 +78,6 @@ def _import_config(config_file=DEFAULT_CONFIG_FILE):
         sys.exit(codes.errors['invalid_yaml_file'])
 
 
-def _make_venv(venv, python, force):
-    """Handles the virtualenv.
-
-    removes the virtualenv if required, else, notifies
-    that it already exists. If it doesn't exist, it will be
-    created.
-    :param string venv: path of virtualenv to install in.
-    :param string python: python binary path to use.
-    :param bool force: whether to force creation or not if it
-     already exists.
-    """
-    if utils.is_virtualenv(venv):
-        if force:
-            lgr.info('Installing within existing virtualenv: {0}'.format(venv))
-        else:
-            lgr.error('Virtualenv already exists at {0}. '
-                      'You can use the -f flag to install within the '
-                      'existing virtualenv.'.format(venv))
-            sys.exit(codes.errors['virtualenv_already_exists'])
-    else:
-        lgr.debug('Creating virtualenv: {0}'.format(venv))
-        utils.make_virtualenv(venv, python)
-
-
 def _handle_output_file(destination_tar, force):
     """Handles the output tar.
 
@@ -333,7 +310,7 @@ def _name_archive(distro, release, version, milestone, build):
 
 
 def create(config=None, config_file=None, force=False, dryrun=False,
-           no_validate=False, verbose=True, virtualenv=None):
+           no_validate=False, verbose=True):
     """Creates an agent package (tar.gz)
 
     This will try to identify the distribution of the host you're running on.
@@ -384,8 +361,8 @@ def create(config=None, config_file=None, force=False, dryrun=False,
             '({0})'.format(ex.message))
         sys.exit(codes.errors['could_not_identify_distribution'])
     python = config.get('python_path', '/usr/bin/python')
-    venv = virtualenv or tempfile.mkdtemp(prefix='agent-packager')
-    venv_already_exists = utils.is_virtualenv(venv)
+    work_root = tempfile.mkdtemp(prefix='agent-packager')
+    venv = os.path.join(work_root, *VENV_ROOT)
     destination_tar = config.get('output_tar', _name_archive(**name_params))
 
     lgr.debug('Distibution is: {0}'.format(name_params['distro']))
@@ -394,7 +371,8 @@ def create(config=None, config_file=None, force=False, dryrun=False,
     lgr.debug('Destination tarfile is: {0}'.format(destination_tar))
 
     if not dryrun:
-        _make_venv(venv, python, force)
+        lgr.debug('Creating virtualenv: {0}'.format(venv))
+        utils.make_virtualenv(venv, python)
 
     _handle_output_file(destination_tar, force)
 
@@ -413,19 +391,18 @@ def create(config=None, config_file=None, force=False, dryrun=False,
     _uninstall_excluded(modules, venv)
     if not no_validate:
         _validate(final_set, venv)
-    utils.tar(venv, destination_tar)
+    utils.tar(work_root, os.path.join(*VENV_ROOT), destination_tar)
 
     lgr.info('The following modules and plugins were installed '
              'in the agent:\n{0}'.format(utils.get_installed(venv)))
 
     # if keep_virtualenv is explicitly specified to be false, the virtualenv
     # will not be deleted.
-    # if keep_virtualenv is not in the config but the virtualenv already
-    # existed, it will not be deleted.
-    if ('keep_virtualenv' in config and not config['keep_virtualenv']) \
-            or ('keep_virtualenv' not in config and not venv_already_exists):
-        lgr.info('Removing origin virtualenv...')
-        shutil.rmtree(venv)
+    if not config.get('keep_virtualenv', False):
+        lgr.info('Removing virtualenv...')
+        shutil.rmtree(work_root)
+    else:
+        lgr.info('Virtualenv kept: {0}'.format(work_root))
 
     # duh!
     lgr.info('Process complete!')

agent_packager/tests/resources/bad_config_file.yaml

@@ -1,3 +1,3 @@
 distribution: Ubuntu
 version: 3.0
- venv: /home/nir0s/Ubuntu-agent/env
+ venv: /home/centos/Ubuntu-agent/env

agent_packager/tests/resources/config_file.yaml

@@ -3,9 +3,9 @@ release: trusty
 python_path: '/usr/bin/python'
 requirements_file: 'agent_packager/tests/resources/requirements.txt'
 # cloudify_agent_version: 3.1
-cloudify_agent_module: https://github.com/nir0s/cloudify-agent/archive/master.tar.gz
+cloudify_agent_module: https://github.com/cloudify-cosmo/cloudify-agent/archive/master.tar.gz
 core_modules:
-    cloudify_plugins_common: https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/3.1.tar.gz
+    cloudify_plugins_common: https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/master.tar.gz
     # cloudify_rest_client: https://github.com/cloudify-cosmo/cloudify-rest-client/archive/3.1.tar.gz
 core_plugins:
     cloudify_script_plugin: exclude
@@ -17,6 +17,6 @@ core_plugins:
 additional_modules:
     - pyyaml==3.10
 additional_plugins:
-    cloudify-fabric-plugin: https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/1.1.tar.gz
+    cloudify-fabric-plugin: https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/1.5.1.tar.gz
 output_tar: Ubuntu-trusty-agent.tar.gz
 keep_virtualenv: true

agent_packager/tests/test_agent_packager.py

@@ -166,7 +166,7 @@ def test_tar(self):
         os.makedirs('dir')
         with open('dir/content.file', 'w') as f:
             f.write('CONTENT')
-        utils.tar('dir', 'tar.file')
+        utils.tar(os.getcwd(), 'dir', 'tar.file')
         shutil.rmtree('dir')
         self.assertTrue(tarfile.is_tarfile('tar.file'))
         with closing(tarfile.open('tar.file', 'r:gz')) as tar:
@@ -176,12 +176,14 @@ def test_tar(self):
 
     @venv
     def test_tar_no_permissions(self):
-        e = self.assertRaises(SystemExit, utils.tar, TEST_VENV, '/file')
+        e = self.assertRaises(SystemExit, utils.tar, os.getcwd(), TEST_VENV,
+                              '/file')
         self.assertEqual(e.message, codes.errors['failed_to_create_tar'])
 
     @venv
     def test_tar_missing_source(self):
-        e = self.assertRaises(SystemExit, utils.tar, 'missing', 'file')
+        e = self.assertRaises(SystemExit, utils.tar, os.getcwd(), 'missing',
+                              'file')
         self.assertEqual(e.message, codes.errors['failed_to_create_tar'])
         os.remove('file')
 
@@ -225,36 +227,6 @@ def test_create_agent_package(self):
             self.assertNotIn(excluded_module, pip_freeze_output)
         shutil.rmtree(TEST_VENV)
 
-    def test_create_agent_package_in_existing_venv_force(self):
-        cli_options = {
-            '--config': CONFIG_FILE,
-            '--force': True,
-            '--dryrun': False,
-            '--no-validation': False,
-            '--verbose': True
-        }
-        utils.make_virtualenv(TEST_VENV)
-        try:
-            cli._run(cli_options)
-        finally:
-            shutil.rmtree(TEST_VENV)
-
-    def test_create_agent_package_in_existing_venv_no_force(self):
-        cli_options = {
-            '--config': CONFIG_FILE,
-            '--force': False,
-            '--dryrun': False,
-            '--no-validation': False,
-            '--verbose': True
-        }
-        utils.make_virtualenv(TEST_VENV)
-        try:
-            e = self.assertRaises(SystemExit, cli._run, cli_options)
-            self.assertEqual(
-                e.message, codes.errors['virtualenv_already_exists'])
-        finally:
-            shutil.rmtree(TEST_VENV)
-
     def test_dryrun(self):
         cli_options = {
             '--config': CONFIG_FILE,
@@ -278,12 +250,6 @@ def test_create_agent_package_no_cloudify_agent_configured(self):
         self.assertEqual(
             e.message, codes.errors['missing_cloudify_agent_config'])
 
-    @venv
-    def test_create_agent_package_existing_venv_no_force(self):
-        e = self.assertRaises(
-            SystemExit, ap.create, None, CONFIG_FILE, verbose=True)
-        self.assertEqual(e.message, codes.errors['virtualenv_already_exists'])
-
     @venv
     def test_create_agent_package_tar_already_exists(self):
         config = ap._import_config(CONFIG_FILE)
@@ -313,7 +279,8 @@ def test_create_agent_package_with_version_info(self):
             ap.create(config, force=True, verbose=True)
             self.assertTrue(os.path.isfile(archive))
         finally:
-            os.remove(archive)
+            if os.path.exists(archive):
+                os.remove(archive)
             os.environ.pop('VERSION')
             os.environ.pop('PRERELEASE')
             os.environ.pop('BUILD')

agent_packager/utils.py

@@ -15,6 +15,7 @@ def run(cmd, no_print=False):
 
     :param string cmd: command to execute
     """
+    lgr.info('Running: {0}'.format(cmd))
     p = subprocess.Popen(
         cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     (stdout, stderr) = p.communicate()
@@ -120,15 +121,16 @@ def download_file(url, destination):
                 f.flush()
 
 
-def tar(source, destination):
+def tar(working_dir, source, destination):
     # TODO: solve or depracate..
     # TODO: apparently, it will tar the first child dir of
     # TODO: source, and not the given parent.
     # with closing(tarfile.open(destination, "w:gz")) as tar:
     #     tar.add(source, arcname=os.path.basename(source))
     # WORKAROUND IMPLEMENTATION
     lgr.info('Creating tar file: {0}'.format(destination))
-    r = run('tar czvf {0} {1}'.format(destination, source), no_print=True)
+    r = run('tar czvf {0} -C {1} {2}'.format(destination, working_dir, source),
+            no_print=False)
     if not r.returncode == 0:
         lgr.error('Failed to create tar file.')
         sys.exit(codes.errors['failed_to_create_tar'])

config/config.yaml

@@ -3,9 +3,9 @@ release: trusty
 python_path: '/usr/bin/python'
 requirements_file: 'agent_packager/tests/resources/requirements.txt'
 # cloudify_agent_version: 3.1
-cloudify_agent_module: https://github.com/nir0s/cloudify-agent/archive/master.tar.gz
+cloudify_agent_module: https://github.com/cloudify-cosmo/cloudify-agent/archive/master.tar.gz
 core_modules:
-    cloudify_plugins_common: https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/3.1.tar.gz
+    cloudify_plugins_common: https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/master.tar.gz
     # cloudify_rest_client: https://github.com/cloudify-cosmo/cloudify-rest-client/archive/3.1.tar.gz
 core_plugins:
     cloudify_script_plugin: exclude
@@ -17,6 +17,6 @@ core_plugins:
 additional_modules:
     - pyyaml==3.10
 additional_plugins:
-    cloudify-fabric-plugin: https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/1.1.tar.gz
+    cloudify-fabric-plugin: https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/1.5.1.tar.gz
 output_tar: Ubuntu-trusty-agent.tar.gz
 keep_virtualenv: true