Merge pull request evansde77#175 from evansde77/feature/nightly-release

Feature/nightly release

src/cirrus/github_tools.py

@@ -92,6 +92,18 @@ def branch_status_list(self, branch):
         for d in data:
             yield d
 
+    def commit_files_optional_push(self, commit_msg, push=True, *filenames):
+        """
+        commit files to the repo, push remote if required.
+
+        """
+        self.repo.index.add(filenames)
+        # commits with message
+        self.repo.index.commit(commit_msg)
+        # push branch to origin
+        if push:
+            return self.repo.remotes.origin.push(self.repo.head)
+
     def log_branch_status(self, branch):
         """
         _log_branch_status_

src/cirrus/package_container.py

@@ -8,6 +8,7 @@
 """
 import sys
 import os
+import git
 import json
 
 from cirrus.logger import get_logger
@@ -61,27 +62,29 @@
 """
 
 LOCAL_INSTALL_SCRIPT = \
-"""
-#!/bin/bash
+"""#!/bin/bash
 
 {virtualenv}
 pip install /opt/{{{{cirrus.configuration.package.name}}}}-{{{{cirrus.configuration.package.version}}}}.tar.gz
 
 """
 
 PYPI_INSTALL_SCRIPT = \
-"""
-#!/bin/bash
+"""#!/bin/bash
 
 {virtualenv}
 pip install {pip_options} {{{{cirrus.configuration.package.name}}}}=={{{{cirrus.configuration.package.version}}}}
 
 """
 
-def make_executable(path):
+
+def make_executable(path, repo):
     mode = os.stat(path).st_mode
     mode |= (mode & 0o444) >> 2    # copy R bits to X
     os.chmod(path, mode)
+    r = git.Repo(repo)
+    r.git.update_index(path, chmod='+x')
+
 
 
 def write_basic_dockerfile(opts, config, path):
@@ -123,15 +126,15 @@ def write_json_file(path, data):
         json.dump(data, handle)
 
 
-def write_script(path, content, **extras):
+def write_script(repo, path, content, **extras):
     """write script content to a file"""
     LOGGER.info("writing script {}".format(path))
 
     script = content.format(**extras)
     with open(path, 'w') as handle:
         handle.write(script)
     # run chmod +x on new script
-    make_executable(path)
+    make_executable(path, repo)
 
 
 def edit_cirrus_conf(opts, config):
@@ -200,26 +203,29 @@ def init_container(opts):
             "excludes": ["post_script.sh", "post_script.sh", ".dockerstache"]
         })
         write_json_file(context, {})
-        write_script(pre_script, DOCKER_PRE_SCRIPT)
+        write_script(opts.repo, pre_script, DOCKER_PRE_SCRIPT)
         write_script(
+            opts.repo,
             local_install,
             LOCAL_INSTALL_SCRIPT,
             virtualenv=venv_option
         )
         write_script(
+            opts.repo,
             pypi_install,
             PYPI_INSTALL_SCRIPT,
             virtualenv=venv_option,
             pip_options=config.pip_options() if config.pip_options() else ""
         )
         if opts.local_install:
             write_script(
+                opts.repo,
                 post_script,
                 DOCKER_POST_SCRIPT,
                 copy_dist=LOCAL_INSTALL_COMMAND.format(package=config.package_name())
             )
         else:
-            write_script(post_script, DOCKER_POST_SCRIPT, copy_dist="")
+            write_script(opts.repo, post_script, DOCKER_POST_SCRIPT, copy_dist="")
         edit_cirrus_conf(opts, config)
 
         modified = [

src/cirrus/plugins/builders/conda_env.py

@@ -29,10 +29,9 @@ def create(self, **kwargs):
         conda = kwargs.get('conda', self.conda_bin)
         upgrade = kwargs.get('upgrade', False)
         nosetupdevelop = kwargs.get('nosetupdevelop', False)
-        environment = kwargs.get(
-            'environment',
-            self.build_config.get('conda-environment', None)
-        )
+        environment = kwargs.get('environment', None)
+        if environment is None:
+            environment = self.build_config.get('conda-environment', None)
         if environment is None:
             msg = "No conda environment yaml specified in cirrus.conf [build] section or via --environment option"
             LOGGER.error(msg)

src/cirrus/release.py

@@ -28,6 +28,7 @@
 from cirrus.plugins.jenkins import JenkinsClient
 from cirrus.req_utils import bump_package
 from cirrus.release_status import release_status
+import cirrus.release_utils as rel_utils
 
 LOGGER = get_logger()
 
@@ -256,6 +257,12 @@ def build_parser(argslist):
         action='store_true',
         dest='major'
     )
+    new_command.add_argument(
+        '--nightly',
+        action='store_true',
+        dest='nightly',
+        default=False
+    )
     new_command.add_argument(
         '--bump',
         nargs=2,
@@ -466,20 +473,22 @@ def new_release(opts):
 
     """
     LOGGER.info("Creating new release...")
-    if not highlander([opts.major, opts.minor, opts.micro]):
-        msg = "Can only specify one of --major, --minor or --micro"
-        LOGGER.error(msg)
-        raise RuntimeError(msg)
-
-    fields = ['major', 'minor', 'micro']
-    mask = [opts.major, opts.minor, opts.micro]
-    field = [x for x in itertools.compress(fields, mask)][0]
-
     config = load_configuration()
-
-    # version bump:
     current_version = config.package_version()
-    new_version = bump_version_field(current_version, field)
+    if opts.nightly:
+        msg = "creating new nightly release..."
+        new_version = rel_utils.new_nightly()
+        field = 'nightly'
+    else:
+        if not highlander([opts.major, opts.minor, opts.micro]):
+            msg = "Can only specify one of --major, --minor or --micro"
+            LOGGER.error(msg)
+            raise RuntimeError(msg)
+
+        fields = ['major', 'minor', 'micro']
+        mask = [opts.major, opts.minor, opts.micro]
+        field = [x for x in itertools.compress(fields, mask)][0]
+        new_version = bump_version_field(current_version, field)
 
     # release branch
     branch_name = "{0}{1}".format(
@@ -786,6 +795,8 @@ def merge_release(opts):
         if not opts.skip_develop:
             ghc.pull_branch(develop, remote=not opts.no_remote)
             ghc.merge_branch(release_branch)
+            if rel_utils.is_nightly(tag):
+                rel_utils.remove_nightly(ghc)
             sha = ghc.repo.head.ref.commit.hexsha
 
             if rel_conf['wait_on_ci_develop']:
@@ -864,6 +875,7 @@ def main():
     opts = build_parser(sys.argv)
     if opts.command == 'new':
         new_release(opts)
+
     if opts.command == 'new-version':
         make_new_version(opts)
 

src/cirrus/release_utils.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+import re
+import datetime
+
+from cirrus.configuration import load_configuration
+
+DEFAULT_FORMAT = "%Y%m%d"
+
+
+def nightly_config(conf=None):
+    """
+    get the nightly config settings from
+    the release section
+
+    """
+    result = {
+        "nightly_format": DEFAULT_FORMAT,
+        "nightly_separator": "-nightly-"
+    }
+    if not conf:
+        conf = load_configuration()
+    if not conf.has_section('release'):
+        return result
+    result['nightly_format'] = conf.get_param("release", "nightly_format", result['nightly_format'])
+    result['nightly_separator'] = conf.get_param("release", "nightly_separator", result['nightly_separator'])
+    return result
+
+
+def is_nightly(version):
+    """
+    return True/False if the version string
+    provided matches a nightly format.
+
+    """
+    conf = nightly_config()
+    reg = "^[0-9]+\.[0-9]+\.[0-9]+{}".format(conf['nightly_separator'])
+    matcher = re.compile(reg)
+    elems = matcher.split(version, 1)
+    if len(elems) == 2:
+        return True
+    return False
+
+
+def new_nightly():
+    """
+    generate a new nightly version
+
+    """
+    cirrus_conf = load_configuration()
+    nightly_conf = nightly_config(cirrus_conf)
+    now = datetime.datetime.now()
+    ts = now.strftime(nightly_conf['nightly_format'])
+    current = cirrus_conf.package_version()
+
+    nightly = "{version}{sep}{ts}".format(
+        version=current,
+        sep=nightly_conf['nightly_separator'],
+        ts=ts
+    )
+    return nightly
+
+
+def remove_nightly(ghc):
+    """
+    remove the nightly part from the cirrus.conf version
+    """
+    cirrus_conf = load_configuration()
+    nightly_conf = nightly_config(cirrus_conf)
+    current = cirrus_conf.package_version()
+    if is_nightly(current):
+        new_version = current.split(nightly_conf['nightly_separator'], 1)[0]
+        cirrus_conf.update_package_version(new_version)
+        ghc.commit_files_optional_push(
+            "remove nightly tag from cirrus.conf",
+            False,
+            "cirrus.conf"
+        )
+    return

tests/unit/cirrus/package_container_tests.py

@@ -58,9 +58,13 @@ def tearDown(self):
         if os.path.exists(self.dir):
             os.system('rm -rf {}'.format(self.dir))
 
-    def test_init_container(self):
+    @mock.patch('cirrus.package_container.git')
+    def test_init_container(self, mock_git):
         """test init_container call"""
-
+        mock_repo = mock.Mock()
+        mock_repo.git = mock.Mock()
+        mock_repo.git.update_index = mock.Mock()
+        mock_git.Repo = mock.Mock(return_value=mock_repo)
         opts = mock.Mock()
         opts.repo = self.dir
         opts.template_dir = "container-template"
@@ -86,6 +90,10 @@ def test_init_container(self):
         found = os.listdir(templates)
         for exp in expected_files:
             self.failUnless(exp in found)
+        self.failUnless(mock_repo.git.update_index.called)
+        for call in mock_repo.git.update_index.call_args_list:
+            self.failUnless(os.path.basename(call[0][0]) in expected_files)
+            self.assertEqual(call[1], {'chmod': '+x'})
 
         dockerfile = os.path.join(templates, 'Dockerfile.mustache')
         with open(dockerfile, 'r') as handle:

tests/unit/cirrus/release_test.py

@@ -36,6 +36,8 @@ def setUp(self):
             )
         self.harness = CirrusConfigurationHarness('cirrus.release.load_configuration', self.config)
         self.harness.setUp()
+        self.harness_utils = CirrusConfigurationHarness('cirrus.release_utils.load_configuration', self.config)
+        self.harness_utils.setUp()
         self.patch_pull = mock.patch('cirrus.release.checkout_and_pull')
         self.patch_branch = mock.patch('cirrus.release.branch')
         self.patch_commit = mock.patch('cirrus.release.commit_files_optional_push')
@@ -48,6 +50,7 @@ def tearDown(self):
         self.patch_branch.stop()
         self.patch_commit.stop()
         self.harness.tearDown()
+        self.harness_utils.tearDown()
         if os.path.exists(self.dir):
             os.system('rm -rf {0}'.format(self.dir))
 
@@ -62,6 +65,7 @@ def test_new_release(self, mock_unstaged):
         opts.micro = True
         opts.major = False
         opts.minor = False
+        opts.nightly = False
         opts.bump = None
 
         # should create a new minor release, editing
@@ -81,6 +85,44 @@ def test_new_release(self, mock_unstaged):
         self.assertEqual(self.mock_commit.call_args[0][2], False)
         self.assertEqual(self.mock_commit.call_args[0][3], 'cirrus.conf')
 
+    @mock.patch('cirrus.release.has_unstaged_changes')
+    @mock.patch('cirrus.release_utils.datetime')
+    def test_new_nightly_release(self, mock_dt, mock_unstaged):
+        """
+        _test_new_release_
+
+        """
+        mock_ts = mock.Mock()
+        mock_ts.strftime = mock.Mock(return_value="TIMESTAMP")
+        mock_now = mock.Mock(return_value=mock_ts)
+        mock_dt.datetime=mock.Mock()
+        mock_dt.datetime.now = mock_now
+        mock_unstaged.return_value = False
+        opts = mock.Mock()
+        opts.micro = False
+        opts.major = False
+        opts.minor = False
+        opts.nightly = True
+        opts.bump = None
+
+        # should create a new minor release, editing
+        # the cirrus config in the test dir
+        new_release(opts)
+
+        # verify new version
+        new_conf = Configuration(self.config)
+        new_conf.load()
+        self.assertEqual(new_conf.package_version(), '1.2.3-nightly-TIMESTAMP')
+
+        self.failUnless(self.mock_pull.called)
+        self.assertEqual(self.mock_pull.call_args[0][1], 'develop')
+        self.failUnless(self.mock_branch.called)
+        self.assertEqual(self.mock_branch.call_args[0][1], 'release/1.2.3-nightly-TIMESTAMP')
+        self.failUnless(self.mock_commit.called)
+        self.assertEqual(self.mock_commit.call_args[0][2], False)
+        self.assertEqual(self.mock_commit.call_args[0][3], 'cirrus.conf')
+
+
     @mock.patch('cirrus.release.has_unstaged_changes')
     @mock.patch('cirrus.release.bump_package')
     def test_new_release_bump(self, mock_bump, mock_unstaged):
@@ -93,6 +135,7 @@ def test_new_release_bump(self, mock_bump, mock_unstaged):
         opts.micro = True
         opts.major = False
         opts.minor = False
+        opts.nightly = False
         opts.bump = [['womp', '1.2.3'], ['wibble', '3.4.5']]
 
         # should create a new minor release, editing
@@ -125,6 +168,7 @@ def test_new_release_unstaged(self, mock_unstaged):
         opts.micro = True
         opts.major = False
         opts.minor = False
+        opts.nightly = False
         opts.bump = None
         self.assertRaises(RuntimeError, new_release, opts)