From f0612472d98c774d47dd6db59b77ef7370601b5f Mon Sep 17 00:00:00 2001 From: devans Date: Thu, 31 Aug 2017 10:37:18 -0500 Subject: [PATCH 1/5] clean newlines --- src/cirrus/package_container.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/cirrus/package_container.py b/src/cirrus/package_container.py index f84628f..91c36c8 100644 --- a/src/cirrus/package_container.py +++ b/src/cirrus/package_container.py @@ -61,8 +61,7 @@ """ LOCAL_INSTALL_SCRIPT = \ -""" -#!/bin/bash +"""#!/bin/bash {virtualenv} pip install /opt/{{{{cirrus.configuration.package.name}}}}-{{{{cirrus.configuration.package.version}}}}.tar.gz @@ -70,14 +69,14 @@ """ PYPI_INSTALL_SCRIPT = \ -""" -#!/bin/bash +"""#!/bin/bash {virtualenv} pip install {pip_options} {{{{cirrus.configuration.package.name}}}}=={{{{cirrus.configuration.package.version}}}} """ + def make_executable(path): mode = os.stat(path).st_mode mode |= (mode & 0o444) >> 2 # copy R bits to X From 232146eaa5a4f7257380e102e493835bf916480d Mon Sep 17 00:00:00 2001 From: devans Date: Thu, 31 Aug 2017 14:34:04 -0500 Subject: [PATCH 2/5] add beta nightly release support --- src/cirrus/github_tools.py | 12 +++ src/cirrus/package_container.py | 19 +++-- src/cirrus/release.py | 36 +++++--- src/cirrus/release_utils.py | 88 ++++++++++++++++++++ tests/unit/cirrus/package_container_tests.py | 12 ++- tests/unit/cirrus/release_test.py | 44 ++++++++++ 6 files changed, 191 insertions(+), 20 deletions(-) create mode 100644 src/cirrus/release_utils.py diff --git a/src/cirrus/github_tools.py b/src/cirrus/github_tools.py index 8494323..fa4e7aa 100644 --- a/src/cirrus/github_tools.py +++ b/src/cirrus/github_tools.py @@ -92,6 +92,18 @@ def branch_status_list(self, branch): for d in data: yield d + def commit_files_optional_push(self, commit_msg, push=True, *filenames): + """ + commit files to the repo, push remote if required. + + """ + self.repo.index.add(filenames) + # commits with message + self.repo.index.commit(commit_msg) + # push branch to origin + if push: + return self.repo.remotes.origin.push(self.repo.head) + def log_branch_status(self, branch): """ _log_branch_status_ diff --git a/src/cirrus/package_container.py b/src/cirrus/package_container.py index 91c36c8..b0f1a9e 100644 --- a/src/cirrus/package_container.py +++ b/src/cirrus/package_container.py @@ -8,6 +8,7 @@ """ import sys import os +import git import json from cirrus.logger import get_logger @@ -77,10 +78,13 @@ """ -def make_executable(path): +def make_executable(path, repo): mode = os.stat(path).st_mode mode |= (mode & 0o444) >> 2 # copy R bits to X os.chmod(path, mode) + r = git.Repo(repo) + r.git.update_index(path, chmod='+x') + def write_basic_dockerfile(opts, config, path): @@ -122,15 +126,15 @@ def write_json_file(path, data): json.dump(data, handle) -def write_script(path, content, **extras): +def write_script(repo, path,content, **extras): """write script content to a file""" LOGGER.info("writing script {}".format(path)) script = content.format(**extras) - with open(path, 'w') as handle: + with open(path, 'wb') as handle: handle.write(script) # run chmod +x on new script - make_executable(path) + make_executable(path, repo) def edit_cirrus_conf(opts, config): @@ -199,13 +203,15 @@ def init_container(opts): "excludes": ["post_script.sh", "post_script.sh", ".dockerstache"] }) write_json_file(context, {}) - write_script(pre_script, DOCKER_PRE_SCRIPT) + write_script(opts.repo, pre_script, DOCKER_PRE_SCRIPT) write_script( + opts.repo, local_install, LOCAL_INSTALL_SCRIPT, virtualenv=venv_option ) write_script( + opts.repo, pypi_install, PYPI_INSTALL_SCRIPT, virtualenv=venv_option, @@ -213,12 +219,13 @@ def init_container(opts): ) if opts.local_install: write_script( + opts.repo, post_script, DOCKER_POST_SCRIPT, copy_dist=LOCAL_INSTALL_COMMAND.format(package=config.package_name()) ) else: - write_script(post_script, DOCKER_POST_SCRIPT, copy_dist="") + write_script(opts.repo, post_script, DOCKER_POST_SCRIPT, copy_dist="") edit_cirrus_conf(opts, config) modified = [ diff --git a/src/cirrus/release.py b/src/cirrus/release.py index 715b28c..65ed7ad 100644 --- a/src/cirrus/release.py +++ b/src/cirrus/release.py @@ -28,6 +28,7 @@ from cirrus.plugins.jenkins import JenkinsClient from cirrus.req_utils import bump_package from cirrus.release_status import release_status +import cirrus.release_utils as rel_utils LOGGER = get_logger() @@ -256,6 +257,12 @@ def build_parser(argslist): action='store_true', dest='major' ) + new_command.add_argument( + '--nightly', + action='store_true', + dest='nightly', + default=False + ) new_command.add_argument( '--bump', nargs=2, @@ -466,20 +473,22 @@ def new_release(opts): """ LOGGER.info("Creating new release...") - if not highlander([opts.major, opts.minor, opts.micro]): - msg = "Can only specify one of --major, --minor or --micro" - LOGGER.error(msg) - raise RuntimeError(msg) - - fields = ['major', 'minor', 'micro'] - mask = [opts.major, opts.minor, opts.micro] - field = [x for x in itertools.compress(fields, mask)][0] - config = load_configuration() + if opts.nightly: + msg = "creating new nightly release..." + new_version = rel_utils.new_nightly() + field = 'nightly' + else: + if not highlander([opts.major, opts.minor, opts.micro]): + msg = "Can only specify one of --major, --minor or --micro" + LOGGER.error(msg) + raise RuntimeError(msg) - # version bump: - current_version = config.package_version() - new_version = bump_version_field(current_version, field) + fields = ['major', 'minor', 'micro'] + mask = [opts.major, opts.minor, opts.micro] + field = [x for x in itertools.compress(fields, mask)][0] + current_version = config.package_version() + new_version = bump_version_field(current_version, field) # release branch branch_name = "{0}{1}".format( @@ -786,6 +795,8 @@ def merge_release(opts): if not opts.skip_develop: ghc.pull_branch(develop, remote=not opts.no_remote) ghc.merge_branch(release_branch) + if rel_utils.is_nightly(tag): + rel_utils.remove_nightly(ghc) sha = ghc.repo.head.ref.commit.hexsha if rel_conf['wait_on_ci_develop']: @@ -864,6 +875,7 @@ def main(): opts = build_parser(sys.argv) if opts.command == 'new': new_release(opts) + if opts.command == 'new-version': make_new_version(opts) diff --git a/src/cirrus/release_utils.py b/src/cirrus/release_utils.py new file mode 100644 index 0000000..b73d51e --- /dev/null +++ b/src/cirrus/release_utils.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +import re +import datetime + +from cirrus.configuration import load_configuration + +DEFAULT_FORMAT = "%Y%m%d" + + +def nightly_config(conf=None): + """ + get the nightly config settings from + the release section + + """ + result = { + "nightly_format": DEFAULT_FORMAT, + "nightly_separator": "-nightly-" + } + if not conf: + conf = load_configuration() + if not conf.has_section('release'): + return result + result['nightly_format'] = conf.get_param("release", "nightly_format", result['nightly_format']) + result['nightly_separator'] = conf.get_param("release", "nightly_separator", result['nightly_separator']) + return result + + +def is_nightly(conf, version): + """ + return True/False if the version string + provided matches a nightly format. + + """ + reg = "^[0-9]+\.[0-9]+\.[0-9]+{}".format(conf['nightly_separator']) + matcher = re.compile(reg) + elems = matcher.split(version, 1) + if len(elems) == 2: + return True + return False + + +def new_nightly(): + """ + generate a new nightly version + + """ + cirrus_conf = load_configuration() + nightly_conf = nightly_config(cirrus_conf) + now = datetime.datetime.now() + ts = now.strftime(nightly_conf['nightly_format']) + current = cirrus_conf.package_version() + + nightly = "{version}{sep}{ts}".format( + version=current, + sep=nightly_conf['nightly_separator'], + ts=ts + ) + return nightly + + +def remove_nightly(ghc): + """ + remove the nightly part from the cirrus.conf version + """ + cirrus_conf = load_configuration() + nightly_conf = nightly_config(cirrus_conf) + current = cirrus_conf.package_version() + if is_nightly(nightly_conf, current): + new_version = current.split(nightly_conf['nightly_separator'], 1)[0] + cirrus_conf.update_package_version(new_version) + ghc.commit_files_optional_push( + "remove nightly tag from cirrus.conf", + False, + "cirrus.conf" + ) + return + + +if __name__ == '__main__': + c = nightly_config() + + + print is_nightly(c, '0.0.0-nightly-20170818') + print is_nightly(c, '0.0.0') + print new_nightly() + diff --git a/tests/unit/cirrus/package_container_tests.py b/tests/unit/cirrus/package_container_tests.py index 25a5fb2..24e9369 100644 --- a/tests/unit/cirrus/package_container_tests.py +++ b/tests/unit/cirrus/package_container_tests.py @@ -58,9 +58,13 @@ def tearDown(self): if os.path.exists(self.dir): os.system('rm -rf {}'.format(self.dir)) - def test_init_container(self): + @mock.patch('cirrus.package_container.git') + def test_init_container(self, mock_git): """test init_container call""" - + mock_repo = mock.Mock() + mock_repo.git = mock.Mock() + mock_repo.git.update_index = mock.Mock() + mock_git.Repo = mock.Mock(return_value=mock_repo) opts = mock.Mock() opts.repo = self.dir opts.template_dir = "container-template" @@ -86,6 +90,10 @@ def test_init_container(self): found = os.listdir(templates) for exp in expected_files: self.failUnless(exp in found) + self.failUnless(mock_repo.git.update_index.called) + for call in mock_repo.git.update_index.call_args_list: + self.failUnless(os.path.basename(call[0][0]) in expected_files) + self.assertEqual(call[1], {'chmod': '+x'}) dockerfile = os.path.join(templates, 'Dockerfile.mustache') with open(dockerfile, 'r') as handle: diff --git a/tests/unit/cirrus/release_test.py b/tests/unit/cirrus/release_test.py index 5abdac9..04eccb4 100644 --- a/tests/unit/cirrus/release_test.py +++ b/tests/unit/cirrus/release_test.py @@ -36,6 +36,8 @@ def setUp(self): ) self.harness = CirrusConfigurationHarness('cirrus.release.load_configuration', self.config) self.harness.setUp() + self.harness_utils = CirrusConfigurationHarness('cirrus.release_utils.load_configuration', self.config) + self.harness_utils.setUp() self.patch_pull = mock.patch('cirrus.release.checkout_and_pull') self.patch_branch = mock.patch('cirrus.release.branch') self.patch_commit = mock.patch('cirrus.release.commit_files_optional_push') @@ -48,6 +50,7 @@ def tearDown(self): self.patch_branch.stop() self.patch_commit.stop() self.harness.tearDown() + self.harness_utils.tearDown() if os.path.exists(self.dir): os.system('rm -rf {0}'.format(self.dir)) @@ -62,6 +65,7 @@ def test_new_release(self, mock_unstaged): opts.micro = True opts.major = False opts.minor = False + opts.nightly = False opts.bump = None # should create a new minor release, editing @@ -81,6 +85,44 @@ def test_new_release(self, mock_unstaged): self.assertEqual(self.mock_commit.call_args[0][2], False) self.assertEqual(self.mock_commit.call_args[0][3], 'cirrus.conf') + @mock.patch('cirrus.release.has_unstaged_changes') + @mock.patch('cirrus.release_utils.datetime') + def test_new_nightly_release(self, mock_dt, mock_unstaged): + """ + _test_new_release_ + + """ + mock_ts = mock.Mock() + mock_ts.strftime = mock.Mock(return_value="TIMESTAMP") + mock_now = mock.Mock(return_value=mock_ts) + mock_dt.datetime=mock.Mock() + mock_dt.datetime.now = mock_now + mock_unstaged.return_value = False + opts = mock.Mock() + opts.micro = False + opts.major = False + opts.minor = False + opts.nightly = True + opts.bump = None + + # should create a new minor release, editing + # the cirrus config in the test dir + new_release(opts) + + # verify new version + new_conf = Configuration(self.config) + new_conf.load() + self.assertEqual(new_conf.package_version(), '1.2.3-nightly-TIMESTAMP') + + self.failUnless(self.mock_pull.called) + self.assertEqual(self.mock_pull.call_args[0][1], 'develop') + self.failUnless(self.mock_branch.called) + self.assertEqual(self.mock_branch.call_args[0][1], 'release/1.2.3-nightly-TIMESTAMP') + self.failUnless(self.mock_commit.called) + self.assertEqual(self.mock_commit.call_args[0][2], False) + self.assertEqual(self.mock_commit.call_args[0][3], 'cirrus.conf') + + @mock.patch('cirrus.release.has_unstaged_changes') @mock.patch('cirrus.release.bump_package') def test_new_release_bump(self, mock_bump, mock_unstaged): @@ -93,6 +135,7 @@ def test_new_release_bump(self, mock_bump, mock_unstaged): opts.micro = True opts.major = False opts.minor = False + opts.nightly = False opts.bump = [['womp', '1.2.3'], ['wibble', '3.4.5']] # should create a new minor release, editing @@ -125,6 +168,7 @@ def test_new_release_unstaged(self, mock_unstaged): opts.micro = True opts.major = False opts.minor = False + opts.nightly = False opts.bump = None self.assertRaises(RuntimeError, new_release, opts) From 22032dbd8652a59d4a631283aa6bedc858f70710 Mon Sep 17 00:00:00 2001 From: devans Date: Thu, 31 Aug 2017 15:04:38 -0500 Subject: [PATCH 3/5] tweaks from testing --- src/cirrus/release.py | 2 +- src/cirrus/release_utils.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cirrus/release.py b/src/cirrus/release.py index 65ed7ad..9dbc9b5 100644 --- a/src/cirrus/release.py +++ b/src/cirrus/release.py @@ -474,6 +474,7 @@ def new_release(opts): """ LOGGER.info("Creating new release...") config = load_configuration() + current_version = config.package_version() if opts.nightly: msg = "creating new nightly release..." new_version = rel_utils.new_nightly() @@ -487,7 +488,6 @@ def new_release(opts): fields = ['major', 'minor', 'micro'] mask = [opts.major, opts.minor, opts.micro] field = [x for x in itertools.compress(fields, mask)][0] - current_version = config.package_version() new_version = bump_version_field(current_version, field) # release branch diff --git a/src/cirrus/release_utils.py b/src/cirrus/release_utils.py index b73d51e..da735de 100644 --- a/src/cirrus/release_utils.py +++ b/src/cirrus/release_utils.py @@ -27,12 +27,13 @@ def nightly_config(conf=None): return result -def is_nightly(conf, version): +def is_nightly(version): """ return True/False if the version string provided matches a nightly format. """ + conf = nightly_config() reg = "^[0-9]+\.[0-9]+\.[0-9]+{}".format(conf['nightly_separator']) matcher = re.compile(reg) elems = matcher.split(version, 1) @@ -67,7 +68,7 @@ def remove_nightly(ghc): cirrus_conf = load_configuration() nightly_conf = nightly_config(cirrus_conf) current = cirrus_conf.package_version() - if is_nightly(nightly_conf, current): + if is_nightly(current): new_version = current.split(nightly_conf['nightly_separator'], 1)[0] cirrus_conf.update_package_version(new_version) ghc.commit_files_optional_push( From ac63a565f16b51781f26f303d2ef1d60e2942cba Mon Sep 17 00:00:00 2001 From: devans Date: Thu, 31 Aug 2017 15:16:27 -0500 Subject: [PATCH 4/5] tweaks for py3 --- src/cirrus/package_container.py | 4 ++-- src/cirrus/release_utils.py | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/cirrus/package_container.py b/src/cirrus/package_container.py index b0f1a9e..8ff2dca 100644 --- a/src/cirrus/package_container.py +++ b/src/cirrus/package_container.py @@ -126,12 +126,12 @@ def write_json_file(path, data): json.dump(data, handle) -def write_script(repo, path,content, **extras): +def write_script(repo, path, content, **extras): """write script content to a file""" LOGGER.info("writing script {}".format(path)) script = content.format(**extras) - with open(path, 'wb') as handle: + with open(path, 'w') as handle: handle.write(script) # run chmod +x on new script make_executable(path, repo) diff --git a/src/cirrus/release_utils.py b/src/cirrus/release_utils.py index da735de..0dec8da 100644 --- a/src/cirrus/release_utils.py +++ b/src/cirrus/release_utils.py @@ -77,13 +77,3 @@ def remove_nightly(ghc): "cirrus.conf" ) return - - -if __name__ == '__main__': - c = nightly_config() - - - print is_nightly(c, '0.0.0-nightly-20170818') - print is_nightly(c, '0.0.0') - print new_nightly() - From 46ff3cc0275274ba46d60adf1ad8425ed96f7906 Mon Sep 17 00:00:00 2001 From: devans Date: Thu, 31 Aug 2017 15:28:28 -0500 Subject: [PATCH 5/5] fix for #174 --- src/cirrus/plugins/builders/conda_env.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/cirrus/plugins/builders/conda_env.py b/src/cirrus/plugins/builders/conda_env.py index b4f303e..3dafbd6 100644 --- a/src/cirrus/plugins/builders/conda_env.py +++ b/src/cirrus/plugins/builders/conda_env.py @@ -29,10 +29,9 @@ def create(self, **kwargs): conda = kwargs.get('conda', self.conda_bin) upgrade = kwargs.get('upgrade', False) nosetupdevelop = kwargs.get('nosetupdevelop', False) - environment = kwargs.get( - 'environment', - self.build_config.get('conda-environment', None) - ) + environment = kwargs.get('environment', None) + if environment is None: + environment = self.build_config.get('conda-environment', None) if environment is None: msg = "No conda environment yaml specified in cirrus.conf [build] section or via --environment option" LOGGER.error(msg)