diff --git a/.gitignore b/.gitignore index 2c62ee1..fb44163 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,6 @@ docs/_build/ *COMMIT_MSG .qbcache/ + +.idea/ + diff --git a/agent_packager/cli.py b/agent_packager/cli.py index a9361c8..ecd79da 100644 --- a/agent_packager/cli.py +++ b/agent_packager/cli.py @@ -9,7 +9,7 @@ Options: -h --help Show this screen -c --config= Path to config yaml (defaults to config.yaml) - -f --force Forces deletion and creation of venv and tar file. + -f --force Forces deletion and creation of tar file. -d --dryrun Prints out the modules to be installed without actually installing them. -n --no-validation Does not validate that all modules were installed correctly. -v --verbose verbose level logging diff --git a/agent_packager/packager.py b/agent_packager/packager.py index 54209f2..c1771fe 100644 --- a/agent_packager/packager.py +++ b/agent_packager/packager.py @@ -6,13 +6,13 @@ import shutil import os import sys +import tempfile import utils import codes DEFAULT_CONFIG_FILE = 'config.yaml' DEFAULT_OUTPUT_TAR_PATH = '{0}-{1}-agent.tar.gz' -DEFAULT_VENV_PATH = 'cloudify/env' PREINSTALL_MODULES = [ 'setuptools==36.8.0' @@ -38,6 +38,7 @@ ] DEFAULT_CLOUDIFY_AGENT_URL = 'https://github.com/cloudify-cosmo/cloudify-agent/archive/{0}.tar.gz' # NOQA +VENV_ROOT = ['cloudify', 'env'] lgr = logger.init() verbose_output = False @@ -62,6 +63,7 @@ def _import_config(config_file=DEFAULT_CONFIG_FILE): :param string config_file: path to config file """ + config_file = os.path.expanduser(config_file) lgr.debug('Importing config: {0}...'.format(config_file)) try: with open(config_file, 'r') as c: @@ -76,30 +78,6 @@ def _import_config(config_file=DEFAULT_CONFIG_FILE): sys.exit(codes.errors['invalid_yaml_file']) -def _make_venv(venv, python, force): - """Handles the virtualenv. - - removes the virtualenv if required, else, notifies - that it already exists. If it doesn't exist, it will be - created. - :param string venv: path of virtualenv to install in. - :param string python: python binary path to use. - :param bool force: whether to force creation or not if it - already exists. - """ - if utils.is_virtualenv(venv): - if force: - lgr.info('Installing within existing virtualenv: {0}'.format(venv)) - else: - lgr.error('Virtualenv already exists at {0}. ' - 'You can use the -f flag to install within the ' - 'existing virtualenv.'.format(venv)) - sys.exit(codes.errors['virtualenv_already_exists']) - else: - lgr.debug('Creating virtualenv: {0}'.format(venv)) - utils.make_virtualenv(venv, python) - - def _handle_output_file(destination_tar, force): """Handles the output tar. @@ -340,8 +318,6 @@ def create(config=None, config_file=None, force=False, dryrun=False, `distribution` (e.g. Ubuntu) config object in the config.yaml. The same goes for the `release` (e.g. Trusty). - A virtualenv will be created under cloudify/env. - The order of the modules' installation is as follows: cloudify-rest-service cloudify-plugins-common @@ -385,8 +361,8 @@ def create(config=None, config_file=None, force=False, dryrun=False, '({0})'.format(ex.message)) sys.exit(codes.errors['could_not_identify_distribution']) python = config.get('python_path', '/usr/bin/python') - venv = DEFAULT_VENV_PATH - venv_already_exists = utils.is_virtualenv(venv) + work_root = tempfile.mkdtemp(prefix='agent-packager') + venv = os.path.join(work_root, *VENV_ROOT) destination_tar = config.get('output_tar', _name_archive(**name_params)) lgr.debug('Distibution is: {0}'.format(name_params['distro'])) @@ -395,7 +371,8 @@ def create(config=None, config_file=None, force=False, dryrun=False, lgr.debug('Destination tarfile is: {0}'.format(destination_tar)) if not dryrun: - _make_venv(venv, python, force) + lgr.debug('Creating virtualenv: {0}'.format(venv)) + utils.make_virtualenv(venv, python) _handle_output_file(destination_tar, force) @@ -414,19 +391,18 @@ def create(config=None, config_file=None, force=False, dryrun=False, _uninstall_excluded(modules, venv) if not no_validate: _validate(final_set, venv) - utils.tar(venv, destination_tar) + utils.tar(work_root, os.path.join(*VENV_ROOT), destination_tar) lgr.info('The following modules and plugins were installed ' 'in the agent:\n{0}'.format(utils.get_installed(venv))) # if keep_virtualenv is explicitly specified to be false, the virtualenv # will not be deleted. - # if keep_virtualenv is not in the config but the virtualenv already - # existed, it will not be deleted. - if ('keep_virtualenv' in config and not config['keep_virtualenv']) \ - or ('keep_virtualenv' not in config and not venv_already_exists): - lgr.info('Removing origin virtualenv...') - shutil.rmtree(venv) + if not config.get('keep_virtualenv', False): + lgr.info('Removing virtualenv...') + shutil.rmtree(work_root) + else: + lgr.info('Virtualenv kept: {0}'.format(work_root)) # duh! lgr.info('Process complete!') diff --git a/agent_packager/tests/resources/bad_config_file.yaml b/agent_packager/tests/resources/bad_config_file.yaml index b2704c3..1f38fda 100644 --- a/agent_packager/tests/resources/bad_config_file.yaml +++ b/agent_packager/tests/resources/bad_config_file.yaml @@ -1,3 +1,3 @@ distribution: Ubuntu version: 3.0 - venv: /home/nir0s/Ubuntu-agent/env \ No newline at end of file + venv: /home/centos/Ubuntu-agent/env \ No newline at end of file diff --git a/agent_packager/tests/resources/config_file.yaml b/agent_packager/tests/resources/config_file.yaml index c9c11ec..4954e30 100644 --- a/agent_packager/tests/resources/config_file.yaml +++ b/agent_packager/tests/resources/config_file.yaml @@ -3,9 +3,9 @@ release: trusty python_path: '/usr/bin/python' requirements_file: 'agent_packager/tests/resources/requirements.txt' # cloudify_agent_version: 3.1 -cloudify_agent_module: https://github.com/nir0s/cloudify-agent/archive/master.tar.gz +cloudify_agent_module: https://github.com/cloudify-cosmo/cloudify-agent/archive/master.tar.gz core_modules: - cloudify_plugins_common: https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/3.1.tar.gz + cloudify_plugins_common: https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/master.tar.gz # cloudify_rest_client: https://github.com/cloudify-cosmo/cloudify-rest-client/archive/3.1.tar.gz core_plugins: cloudify_script_plugin: exclude @@ -17,6 +17,6 @@ core_plugins: additional_modules: - pyyaml==3.10 additional_plugins: - cloudify-fabric-plugin: https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/1.1.tar.gz + cloudify-fabric-plugin: https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/1.5.1.tar.gz output_tar: Ubuntu-trusty-agent.tar.gz keep_virtualenv: true \ No newline at end of file diff --git a/agent_packager/tests/test_agent_packager.py b/agent_packager/tests/test_agent_packager.py index aed9724..7cf52b8 100644 --- a/agent_packager/tests/test_agent_packager.py +++ b/agent_packager/tests/test_agent_packager.py @@ -166,7 +166,7 @@ def test_tar(self): os.makedirs('dir') with open('dir/content.file', 'w') as f: f.write('CONTENT') - utils.tar('dir', 'tar.file') + utils.tar(os.getcwd(), 'dir', 'tar.file') shutil.rmtree('dir') self.assertTrue(tarfile.is_tarfile('tar.file')) with closing(tarfile.open('tar.file', 'r:gz')) as tar: @@ -176,12 +176,14 @@ def test_tar(self): @venv def test_tar_no_permissions(self): - e = self.assertRaises(SystemExit, utils.tar, TEST_VENV, '/file') + e = self.assertRaises(SystemExit, utils.tar, os.getcwd(), TEST_VENV, + '/file') self.assertEqual(e.message, codes.errors['failed_to_create_tar']) @venv def test_tar_missing_source(self): - e = self.assertRaises(SystemExit, utils.tar, 'missing', 'file') + e = self.assertRaises(SystemExit, utils.tar, os.getcwd(), 'missing', + 'file') self.assertEqual(e.message, codes.errors['failed_to_create_tar']) os.remove('file') @@ -225,36 +227,6 @@ def test_create_agent_package(self): self.assertNotIn(excluded_module, pip_freeze_output) shutil.rmtree(TEST_VENV) - def test_create_agent_package_in_existing_venv_force(self): - cli_options = { - '--config': CONFIG_FILE, - '--force': True, - '--dryrun': False, - '--no-validation': False, - '--verbose': True - } - utils.make_virtualenv(TEST_VENV) - try: - cli._run(cli_options) - finally: - shutil.rmtree(TEST_VENV) - - def test_create_agent_package_in_existing_venv_no_force(self): - cli_options = { - '--config': CONFIG_FILE, - '--force': False, - '--dryrun': False, - '--no-validation': False, - '--verbose': True - } - utils.make_virtualenv(TEST_VENV) - try: - e = self.assertRaises(SystemExit, cli._run, cli_options) - self.assertEqual( - e.message, codes.errors['virtualenv_already_exists']) - finally: - shutil.rmtree(TEST_VENV) - def test_dryrun(self): cli_options = { '--config': CONFIG_FILE, @@ -278,12 +250,6 @@ def test_create_agent_package_no_cloudify_agent_configured(self): self.assertEqual( e.message, codes.errors['missing_cloudify_agent_config']) - @venv - def test_create_agent_package_existing_venv_no_force(self): - e = self.assertRaises( - SystemExit, ap.create, None, CONFIG_FILE, verbose=True) - self.assertEqual(e.message, codes.errors['virtualenv_already_exists']) - @venv def test_create_agent_package_tar_already_exists(self): config = ap._import_config(CONFIG_FILE) @@ -313,7 +279,8 @@ def test_create_agent_package_with_version_info(self): ap.create(config, force=True, verbose=True) self.assertTrue(os.path.isfile(archive)) finally: - os.remove(archive) + if os.path.exists(archive): + os.remove(archive) os.environ.pop('VERSION') os.environ.pop('PRERELEASE') os.environ.pop('BUILD') diff --git a/agent_packager/utils.py b/agent_packager/utils.py index b54a90e..f71dc4e 100644 --- a/agent_packager/utils.py +++ b/agent_packager/utils.py @@ -15,6 +15,7 @@ def run(cmd, no_print=False): :param string cmd: command to execute """ + lgr.info('Running: {0}'.format(cmd)) p = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() @@ -120,7 +121,7 @@ def download_file(url, destination): f.flush() -def tar(source, destination): +def tar(working_dir, source, destination): # TODO: solve or depracate.. # TODO: apparently, it will tar the first child dir of # TODO: source, and not the given parent. @@ -128,7 +129,8 @@ def tar(source, destination): # tar.add(source, arcname=os.path.basename(source)) # WORKAROUND IMPLEMENTATION lgr.info('Creating tar file: {0}'.format(destination)) - r = run('tar czvf {0} {1}'.format(destination, source), no_print=True) + r = run('tar czvf {0} -C {1} {2}'.format(destination, working_dir, source), + no_print=False) if not r.returncode == 0: lgr.error('Failed to create tar file.') sys.exit(codes.errors['failed_to_create_tar']) diff --git a/config/config.yaml b/config/config.yaml index c9c11ec..4954e30 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -3,9 +3,9 @@ release: trusty python_path: '/usr/bin/python' requirements_file: 'agent_packager/tests/resources/requirements.txt' # cloudify_agent_version: 3.1 -cloudify_agent_module: https://github.com/nir0s/cloudify-agent/archive/master.tar.gz +cloudify_agent_module: https://github.com/cloudify-cosmo/cloudify-agent/archive/master.tar.gz core_modules: - cloudify_plugins_common: https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/3.1.tar.gz + cloudify_plugins_common: https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/master.tar.gz # cloudify_rest_client: https://github.com/cloudify-cosmo/cloudify-rest-client/archive/3.1.tar.gz core_plugins: cloudify_script_plugin: exclude @@ -17,6 +17,6 @@ core_plugins: additional_modules: - pyyaml==3.10 additional_plugins: - cloudify-fabric-plugin: https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/1.1.tar.gz + cloudify-fabric-plugin: https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/1.5.1.tar.gz output_tar: Ubuntu-trusty-agent.tar.gz keep_virtualenv: true \ No newline at end of file