From 06f93cff1722ddf79af06a2a77a84dfb05134c34 Mon Sep 17 00:00:00 2001 From: Isaac Shabtay Date: Thu, 8 Mar 2018 05:59:15 -0800 Subject: [PATCH] CFY-7569: Use temporary directory instead of current directory for temp files --- agent_packager/cli.py | 8 ++-- agent_packager/packager.py | 47 +++++-------------- .../tests/resources/bad_config_file.yaml | 2 +- .../tests/resources/config_file.yaml | 6 +-- agent_packager/tests/test_agent_packager.py | 47 +++---------------- agent_packager/utils.py | 6 ++- config/config.yaml | 2 +- 7 files changed, 31 insertions(+), 87 deletions(-) diff --git a/agent_packager/cli.py b/agent_packager/cli.py index 12c92c2..ecd79da 100644 --- a/agent_packager/cli.py +++ b/agent_packager/cli.py @@ -3,16 +3,15 @@ """Script to run Cloudify's Agent Packager via command line Usage: - cfy-ap [--config= --force --dryrun --no-validation -v --env=] + cfy-ap [--config= --force --dryrun --no-validation -v] cfy-ap --version Options: -h --help Show this screen -c --config= Path to config yaml (defaults to config.yaml) - -f --force Forces deletion and creation of venv and tar file. + -f --force Forces deletion and creation of tar file. -d --dryrun Prints out the modules to be installed without actually installing them. -n --no-validation Does not validate that all modules were installed correctly. - -e --env= Virtualenv to use (defaults to a temporary directory) -v --verbose verbose level logging --version Display current version """ @@ -48,8 +47,7 @@ def _run(test_options=None): force=options.get('--force'), dryrun=options.get('--dryrun'), no_validate=options.get('--no-validation'), - verbose=options.get('--verbose'), - virtualenv=options.get('--env') + verbose=options.get('--verbose') ) def main(): diff --git a/agent_packager/packager.py b/agent_packager/packager.py index 16daa64..c1771fe 100644 --- a/agent_packager/packager.py +++ b/agent_packager/packager.py @@ -38,6 +38,7 @@ ] DEFAULT_CLOUDIFY_AGENT_URL = 'https://github.com/cloudify-cosmo/cloudify-agent/archive/{0}.tar.gz' # NOQA +VENV_ROOT = ['cloudify', 'env'] lgr = logger.init() verbose_output = False @@ -77,30 +78,6 @@ def _import_config(config_file=DEFAULT_CONFIG_FILE): sys.exit(codes.errors['invalid_yaml_file']) -def _make_venv(venv, python, force): - """Handles the virtualenv. - - removes the virtualenv if required, else, notifies - that it already exists. If it doesn't exist, it will be - created. - :param string venv: path of virtualenv to install in. - :param string python: python binary path to use. - :param bool force: whether to force creation or not if it - already exists. - """ - if utils.is_virtualenv(venv): - if force: - lgr.info('Installing within existing virtualenv: {0}'.format(venv)) - else: - lgr.error('Virtualenv already exists at {0}. ' - 'You can use the -f flag to install within the ' - 'existing virtualenv.'.format(venv)) - sys.exit(codes.errors['virtualenv_already_exists']) - else: - lgr.debug('Creating virtualenv: {0}'.format(venv)) - utils.make_virtualenv(venv, python) - - def _handle_output_file(destination_tar, force): """Handles the output tar. @@ -333,7 +310,7 @@ def _name_archive(distro, release, version, milestone, build): def create(config=None, config_file=None, force=False, dryrun=False, - no_validate=False, verbose=True, virtualenv=None): + no_validate=False, verbose=True): """Creates an agent package (tar.gz) This will try to identify the distribution of the host you're running on. @@ -384,8 +361,8 @@ def create(config=None, config_file=None, force=False, dryrun=False, '({0})'.format(ex.message)) sys.exit(codes.errors['could_not_identify_distribution']) python = config.get('python_path', '/usr/bin/python') - venv = virtualenv or tempfile.mkdtemp(prefix='agent-packager') - venv_already_exists = utils.is_virtualenv(venv) + work_root = tempfile.mkdtemp(prefix='agent-packager') + venv = os.path.join(work_root, *VENV_ROOT) destination_tar = config.get('output_tar', _name_archive(**name_params)) lgr.debug('Distibution is: {0}'.format(name_params['distro'])) @@ -394,7 +371,8 @@ def create(config=None, config_file=None, force=False, dryrun=False, lgr.debug('Destination tarfile is: {0}'.format(destination_tar)) if not dryrun: - _make_venv(venv, python, force) + lgr.debug('Creating virtualenv: {0}'.format(venv)) + utils.make_virtualenv(venv, python) _handle_output_file(destination_tar, force) @@ -413,19 +391,18 @@ def create(config=None, config_file=None, force=False, dryrun=False, _uninstall_excluded(modules, venv) if not no_validate: _validate(final_set, venv) - utils.tar(venv, destination_tar) + utils.tar(work_root, os.path.join(*VENV_ROOT), destination_tar) lgr.info('The following modules and plugins were installed ' 'in the agent:\n{0}'.format(utils.get_installed(venv))) # if keep_virtualenv is explicitly specified to be false, the virtualenv # will not be deleted. - # if keep_virtualenv is not in the config but the virtualenv already - # existed, it will not be deleted. - if ('keep_virtualenv' in config and not config['keep_virtualenv']) \ - or ('keep_virtualenv' not in config and not venv_already_exists): - lgr.info('Removing origin virtualenv...') - shutil.rmtree(venv) + if not config.get('keep_virtualenv', False): + lgr.info('Removing virtualenv...') + shutil.rmtree(work_root) + else: + lgr.info('Virtualenv kept: {0}'.format(work_root)) # duh! lgr.info('Process complete!') diff --git a/agent_packager/tests/resources/bad_config_file.yaml b/agent_packager/tests/resources/bad_config_file.yaml index b2704c3..1f38fda 100644 --- a/agent_packager/tests/resources/bad_config_file.yaml +++ b/agent_packager/tests/resources/bad_config_file.yaml @@ -1,3 +1,3 @@ distribution: Ubuntu version: 3.0 - venv: /home/nir0s/Ubuntu-agent/env \ No newline at end of file + venv: /home/centos/Ubuntu-agent/env \ No newline at end of file diff --git a/agent_packager/tests/resources/config_file.yaml b/agent_packager/tests/resources/config_file.yaml index c9c11ec..4954e30 100644 --- a/agent_packager/tests/resources/config_file.yaml +++ b/agent_packager/tests/resources/config_file.yaml @@ -3,9 +3,9 @@ release: trusty python_path: '/usr/bin/python' requirements_file: 'agent_packager/tests/resources/requirements.txt' # cloudify_agent_version: 3.1 -cloudify_agent_module: https://github.com/nir0s/cloudify-agent/archive/master.tar.gz +cloudify_agent_module: https://github.com/cloudify-cosmo/cloudify-agent/archive/master.tar.gz core_modules: - cloudify_plugins_common: https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/3.1.tar.gz + cloudify_plugins_common: https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/master.tar.gz # cloudify_rest_client: https://github.com/cloudify-cosmo/cloudify-rest-client/archive/3.1.tar.gz core_plugins: cloudify_script_plugin: exclude @@ -17,6 +17,6 @@ core_plugins: additional_modules: - pyyaml==3.10 additional_plugins: - cloudify-fabric-plugin: https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/1.1.tar.gz + cloudify-fabric-plugin: https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/1.5.1.tar.gz output_tar: Ubuntu-trusty-agent.tar.gz keep_virtualenv: true \ No newline at end of file diff --git a/agent_packager/tests/test_agent_packager.py b/agent_packager/tests/test_agent_packager.py index aed9724..7cf52b8 100644 --- a/agent_packager/tests/test_agent_packager.py +++ b/agent_packager/tests/test_agent_packager.py @@ -166,7 +166,7 @@ def test_tar(self): os.makedirs('dir') with open('dir/content.file', 'w') as f: f.write('CONTENT') - utils.tar('dir', 'tar.file') + utils.tar(os.getcwd(), 'dir', 'tar.file') shutil.rmtree('dir') self.assertTrue(tarfile.is_tarfile('tar.file')) with closing(tarfile.open('tar.file', 'r:gz')) as tar: @@ -176,12 +176,14 @@ def test_tar(self): @venv def test_tar_no_permissions(self): - e = self.assertRaises(SystemExit, utils.tar, TEST_VENV, '/file') + e = self.assertRaises(SystemExit, utils.tar, os.getcwd(), TEST_VENV, + '/file') self.assertEqual(e.message, codes.errors['failed_to_create_tar']) @venv def test_tar_missing_source(self): - e = self.assertRaises(SystemExit, utils.tar, 'missing', 'file') + e = self.assertRaises(SystemExit, utils.tar, os.getcwd(), 'missing', + 'file') self.assertEqual(e.message, codes.errors['failed_to_create_tar']) os.remove('file') @@ -225,36 +227,6 @@ def test_create_agent_package(self): self.assertNotIn(excluded_module, pip_freeze_output) shutil.rmtree(TEST_VENV) - def test_create_agent_package_in_existing_venv_force(self): - cli_options = { - '--config': CONFIG_FILE, - '--force': True, - '--dryrun': False, - '--no-validation': False, - '--verbose': True - } - utils.make_virtualenv(TEST_VENV) - try: - cli._run(cli_options) - finally: - shutil.rmtree(TEST_VENV) - - def test_create_agent_package_in_existing_venv_no_force(self): - cli_options = { - '--config': CONFIG_FILE, - '--force': False, - '--dryrun': False, - '--no-validation': False, - '--verbose': True - } - utils.make_virtualenv(TEST_VENV) - try: - e = self.assertRaises(SystemExit, cli._run, cli_options) - self.assertEqual( - e.message, codes.errors['virtualenv_already_exists']) - finally: - shutil.rmtree(TEST_VENV) - def test_dryrun(self): cli_options = { '--config': CONFIG_FILE, @@ -278,12 +250,6 @@ def test_create_agent_package_no_cloudify_agent_configured(self): self.assertEqual( e.message, codes.errors['missing_cloudify_agent_config']) - @venv - def test_create_agent_package_existing_venv_no_force(self): - e = self.assertRaises( - SystemExit, ap.create, None, CONFIG_FILE, verbose=True) - self.assertEqual(e.message, codes.errors['virtualenv_already_exists']) - @venv def test_create_agent_package_tar_already_exists(self): config = ap._import_config(CONFIG_FILE) @@ -313,7 +279,8 @@ def test_create_agent_package_with_version_info(self): ap.create(config, force=True, verbose=True) self.assertTrue(os.path.isfile(archive)) finally: - os.remove(archive) + if os.path.exists(archive): + os.remove(archive) os.environ.pop('VERSION') os.environ.pop('PRERELEASE') os.environ.pop('BUILD') diff --git a/agent_packager/utils.py b/agent_packager/utils.py index b54a90e..f71dc4e 100644 --- a/agent_packager/utils.py +++ b/agent_packager/utils.py @@ -15,6 +15,7 @@ def run(cmd, no_print=False): :param string cmd: command to execute """ + lgr.info('Running: {0}'.format(cmd)) p = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() @@ -120,7 +121,7 @@ def download_file(url, destination): f.flush() -def tar(source, destination): +def tar(working_dir, source, destination): # TODO: solve or depracate.. # TODO: apparently, it will tar the first child dir of # TODO: source, and not the given parent. @@ -128,7 +129,8 @@ def tar(source, destination): # tar.add(source, arcname=os.path.basename(source)) # WORKAROUND IMPLEMENTATION lgr.info('Creating tar file: {0}'.format(destination)) - r = run('tar czvf {0} {1}'.format(destination, source), no_print=True) + r = run('tar czvf {0} -C {1} {2}'.format(destination, working_dir, source), + no_print=False) if not r.returncode == 0: lgr.error('Failed to create tar file.') sys.exit(codes.errors['failed_to_create_tar']) diff --git a/config/config.yaml b/config/config.yaml index c9c11ec..37c4acb 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -3,7 +3,7 @@ release: trusty python_path: '/usr/bin/python' requirements_file: 'agent_packager/tests/resources/requirements.txt' # cloudify_agent_version: 3.1 -cloudify_agent_module: https://github.com/nir0s/cloudify-agent/archive/master.tar.gz +cloudify_agent_module: https://github.com/cloudify-cosmo/cloudify-agent/archive/master.tar.gz core_modules: cloudify_plugins_common: https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/3.1.tar.gz # cloudify_rest_client: https://github.com/cloudify-cosmo/cloudify-rest-client/archive/3.1.tar.gz