From f6ed674a73a5f6537643b5098c2ddb5c3795b13c Mon Sep 17 00:00:00 2001 From: Dave Evans Date: Thu, 24 Aug 2017 08:03:26 -0500 Subject: [PATCH 1/9] Update README.md --- README.md | 66 ++----------------------------------------------------- 1 file changed, 2 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 10e24c3..92560ea 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,7 @@ curl -O https://raw.githubusercontent.com/evansde77/cirrus/develop/conda-install bash conda-installer.sh ``` -The installer script will set up an install of cirrus for you in your home directory -and prompt for some info so that it can set up some parameters in your .gitconfig -The installer will create a virtualenv and install cirrus from pip via the cirrus-cli package, installing the latest available version. - +See the Installation for Development: ============================= @@ -49,6 +46,7 @@ cd cirrus git cirrus build ``` +For more detailed docs see the [Installation Docs](https://github.com/evansde77/cirrus/wiki/Installation) User Configuration File: ======================== @@ -57,11 +55,7 @@ As cirrus works as a git extension, it will use your gitconfig file. The install 1. *github-user* - Your Github Username 1. *github-token* - A github access token -1. *pypi-user* - Username of your pypi server -1. *pypi-token* - Token/password to access your pypi server -1. *pypi-ssh-key* - SSH key used for scp-like uploads to your pypi server (if HTTP upload isnt supported) -*Protip:* If you require a different username for ssh access to your pypi server, you can add an optional *pypi-ssh-user* setting. Package Configuration Files: ============================ @@ -81,62 +75,6 @@ Usage: git cirrus hello ``` -#### cirrus package - -`Beta: new in Release 1.6 and above` - -The cirrus package command provides support for setting up cirrus to work in a new package. -Assuming a starting point of a git repo with at least one commit on the master branch, you can run the git cirrus package init command -to add the various configuration and packaging files used by cirrus. This is intended to provide a minimal working template, so some configuration post init will be needed to take advantage of other features. - -The init command will do the following: - -1. set up the develop/master branches for gitflow style work -2. generate a MANIFEST.in file containing the necessary include statements for packaging with setuptools -3. adds the default cirrus setup.py file -4. Generates a basic cirrus.conf file for the package -5. sets up the version number management and history file generation - -For example: -```bash -# create a minimal repo -mkdir test_repo -cd test_repo -git init -echo 'readme' > README -git add README -git commit -m 'add README; first commit on master' -git checkout -b develop -mkdir -p src/throwaway -echo "__version__ = '0.0.0'" > src/throwaway/__init__.py -echo "requests" > requirements.txt -git add src/throwaway/__init__.py requirements.txt -git commit -m "make a package" -git checkout master -git merge develop - -# run the package init command -git cirrus package init -p test_repo --no-remote -v 0.0.0 -s src --version-file=src/throwaway/__init__.py -# can now use cirrus in the package, eg build: -git cirrus build - -``` - -Options available for the package init command: - -* --repo, -r - Path to repo, defaults to pwd -* --source-dir -s Additional directory structure within package for source code eg if you put your source code in a src subdir in the repo. Assumes repo top level directory if not set -* --package, -p cirrus package name (Required) -* --version, -v initial package version -* --organization, -o Organization name to include in package (Eg, Github org or username) -* --description, -d Package description -* --templates, additional template rules to include in MANIFEST, eg include src/templates/*.json -* --version-file, Version file, defaults to package __init__.py, where to update the \_\_version\_\_ attribute -* --history-file, changelog history filename, defaults to HISTORY.md -* --requirements, requirements file for pip defaults to requirements.txt -* --master-branch, GitFlow master branch, defaults to master -* --develop-branch, GitFlow develop branch, defaults to develop -* --no-remote, disable pushing changes to remote, commit locally only #### cirrus build From 6d6aa013cdc95df4daaa9938710adb9b828c0783 Mon Sep 17 00:00:00 2001 From: Dave Evans Date: Thu, 24 Aug 2017 08:09:53 -0500 Subject: [PATCH 2/9] Update README.md --- README.md | 154 +++--------------------------------------------------- 1 file changed, 8 insertions(+), 146 deletions(-) diff --git a/README.md b/README.md index 92560ea..2222d0c 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,8 @@ This file, coupled with the cirrus setup.py template and command line tools dict Cirrus Commands: ================ +See the [Cirrus Commands Docs](https://github.com/evansde77/cirrus/wiki#command-reference) + #### cirrus hello A simple test command that says hello, verifies that things are working and prints out some info about your cirrus install @@ -76,43 +78,15 @@ git cirrus hello ``` - -#### cirrus build -Builds a new virtualenv and installs the requirements for the package, setting up a development/testing/deployment environment for the package. - -Usage: -```bash -git cirrus build -``` - -The virtualenv is created in ./venv. -Optional parameters for the build command are read from the cirrus.conf, they are; - -1. build Section - 1. virtualenv_name - sets the name of the virtualenv directory, defaults to venv - 2. requirements_file - name of the requirements.txt file, defaults to requirements.txt -2. pypi Section - 1. pypi_url - If present, will use the pypi server to install requirements, also requires the pypi username and token to be set in the cirrus section of your gitconfig -3. Other options - 1. `-c, --clean` removes the existing ./venv before building. - 2. `-d, --docs` generate documentation using Sphinx and its generated Makefile. Any optional make commands can be passed along (e.g., --docs clean singlehtml) - 1. Requires a `sphinx_makefile_dir` value set in the `docs` section of cirrus.conf. - 2. `sphinx_makefile_dir` should point to the directory that contains Sphinx's Makefile. +* [git cirrus build](https://github.com/evansde77/cirrus/wiki/BuildCommand) +* [git cirrus test](https://github.com/evansde77/cirrus/wiki/TestCommand) +* [git cirrus release](https://github.com/evansde77/cirrus/wiki/ReleaseCommand) +* [git cirrus feature](https://github.com/evansde77/cirrus/wiki/FeatureCommand) +* [git cirrus docker-image](https://github.com/evansde77/cirrus/wiki/DockerImageCommand) +* [git cirrus selfupdate](https://github.com/evansde77/cirrus/wiki/SelfupdateCommand) -#### cirrus feature -Commands related to the creatation and management of a git-flow style feature branch. -1. new - Creates a new feature branch, optionally pushing the new branch upstream following a git-flow style workflow -2. pull-request - Creates a new Pull Request in github requesting to merge the current feature branch with the develop branch, specifying the title, body and list of people to tag in the PR. -3. pr - shorthand for pull-request -4. list - lists all open unmerged feature branches in the repo - -Usage: -```bash -git cirrus feature new BRANCH_NAME --push -git cirrus feature pull-request --title TITLE --body BODY --notify @AGITHUBUSER,@ANOTHERGITHUBUSER -``` #### cirrus review The cirrus review command provides some utilities for dealing with GitHub pull requests from the cirrus command line. @@ -133,105 +107,9 @@ git cirrus review plusone --id 500 -c "+1" # adds the +1 context to the fea git cirrus review reviee --id 500 -m "great work, LGTM" --plus-one -c "+1" # adds a comment to the PR and sets the +1 context status to success ``` -#### cirrus release -Commands related to creation of a new git-flow style release branch, building the release and uploading it to a pypi server. -There are three subcommands: - -1. new - creates a new release branch, increments the package version, builds the release notes if configured. -2. build - Runs sdist to create a new build artifact from the release branch -3. merge - Runs git-flow style branch merges back to master and develop, optionally waiting on CI or setting flags for GH build contexts if needed -4. upload - Pushes the build artifact to the pypi server configured in the cirrus conf, using a plugin system to allow for customisation. - -Usage: -```bash -git cirrus test # Stop if things are broken -git cirrus release new --micro -git cirrus release build -git cirrus release merge --cleanup -git cirrus release upload --plugin pypi -``` - -Options: - -1. release new requires one of --micro, --minor or --macro to indicate which semantic version field to increment -2. --bump adds or updates a package==version pair in requirements.txt, e.g. `--bump foo==0.0.9 bar==1.2.3`. -3. release merge supports the following options: - * --cleanup - removes the remote and local release branch on successful merge - * --context-string - Update the github context string provided when pushed - * --wait-on-ci - Wait for GitHub CI status to be success before uploading -4. upload will push the new release and upload the build artifact to pypi, but may take several non-required options: - * --plugin - Name of the upload plugin module. Options are found in [https://github.com/evansde77/cirrus/tree/develop/src/cirrus/plugins/uploaders](cirrus/plugins/uploaders) and can be used to customise the upload process. The pypi plugin does a standard sdist upload to the pypi server configured in your pypirc. The fabric plugin uses fabric to scp the artifact to a custom pypi server. - * --test do not push new release or upload build artifact to pypi - * --pypi-sudo, --no-pypi-sudo use or do not use sudo to move the build artifact to the correct location in the pypi server, defaults to using sudo - * --pypi-url URL override the pypi url from cirrus.conf with URL, equivalent to the -r option for pypi.python.org uploads, can specify a url or a shorthand name from your pypirc. - - -Release Options in cirrus.conf - -You can customise the release merge workflow for each package via the cirrus config with the following settings: - - * wait_on_ci - Set true to enable waiting on CI builds for the release branch, defaults to False, this will make the merge command poll the GH status API for the commit and wait for it to become success - * wait_on_ci_develop - Set true to enable waiting on CI builds for the develop branch, defaults to False - * wait_on_ci_master - Set true to enable waiting on CI builds for the master branch, defaults to False - * wait_on_ci_timeout - Timeout in seconds to give up on waiting on CI, defaults to 600s (10 mins) - * wait_on_ci_interval - Interval to poll status in seconds, defaults to 2 seconds - * github_context_string - The github context to update status for if update_github_context is True Eg: continuous-integration/travis-ci - * update_github_context - An alternative to waiting on CI, you can simply flip the status for a context to success if eg you have protected branches without a CI build to wait for. Requires a context to be provided via the github_context_string setting - * push_retry_attempts - Optional number of attempts to try to push during merge - * push_retry_cooloff - Optional time to wait between retries in seconds - -Example: - -```ini -[release] -wait_on_ci = False -wait_on_ci_develop = False -wait_on_ci_master = False -wait_on_ci_timeout = 600 -wait_on_ci_interval = 2 -github_context_string = continuous-integration/travis-ci -update_github_context = True -push_retry_attempts = 10 -push_retry_cooloff = 2 -``` - -##### Working with GitHub protected branches - -The release command works with two possible mode of [https://github.com/blog/2051-protected-branches-and-required-status-checks](GitHub branch protection) one in which you wait for the CI tests to run and update the status, and one in which there are no checks, so that you have to set status to merge branches in a git flow style. - -For the waiting mode, simply flip the boolean wait_on_ci flag to True to wait on CI to run on the release branch. Likewise, the wait_on_ci_develop and wait_on_ci_master params will wait on CI status for the develop and master branches you have configured before merging. You can adjust the total timeout in seconds and the poll interval via the cirrus config file. - -For the non-waiting mode, you provide the github context string you want to set and then when the merges back to develop and master occur, that state value is set to success. This mode of operation is useful when your CI system runs release tests in a way not connected to github, but you still have protected branches. - -##### release tips - -*Protip:* If you don't make releases regularly, you'll want to make sure your local repo copy is up to date (cirrus should do these eventually). -``` -git checkout master # get master, if you only have develop -git fetch -git fetch --tags -git checkout develop # you're ready to release! -``` - -*Protip:* If something goes wrong during release building you may end up on a release/A.B.C branch that didn't work out. If you haven't pushed out the tag for the new version you can `git checkout develop` and `git branch -d release/A.B.C`. If you've pushed a bad version/tag the best thing to do is resolve the problem and create a new micro version -- don't modify a tag that's already remote (consult your own release workflow). - -#### cirrus test -Command for running tests in a package. - -Usage: -```bash -git cirrus test -``` - -Options and config: ---suite SUITE_LOCATION - -Must define name of virtualenv in [package] virtualenv -Must define [test-default] where (default location for tests, optional if you choose to always include --suite) -May define [test-SUITE_LOCATION] where #### cirrus qc @@ -248,22 +126,6 @@ Specific files may be ran using '--files' OR check only files that have not yet For pylint, a score threshold must be set in cirrus.conf [quality] threshold. The path to an optional rcfile (pylint configuration) may be set at [quality] rcfile. -#### cirrus deploy - -The deploy command provides a plugin driven way to hook into deployment systems like chef and puppet. -Since deployment is heavily customisable based on what system is in use, the command supports selecting a plugin and then delegates all CLI options to that plugin. - -Usage: -```bash -git cirrus deploy --plugin= -``` - -Deployer plugins live in [cirrus.plugins.deployers](https://github.com/evansde77/cirrus/tree/develop/src/cirrus/plugins/deployers) and individual docs for each plugin can be found there. - -Plugins: - - * [chef](https://github.com/evansde77/cirrus/blob/develop/src/cirrus/plugins/deployers/chef.py) - [docs](https://github.com/evansde77/cirrus/blob/develop/src/cirrus/plugins/deployers/chef.md) - #### cirrus docs Command for publishing Sphinx documentation From 21c12c54bc7746e38cc150fff78bf58e9e0f86be Mon Sep 17 00:00:00 2001 From: Dave Evans Date: Thu, 24 Aug 2017 10:17:45 -0500 Subject: [PATCH 3/9] Update README.md --- README.md | 146 ++++-------------------------------------------------- 1 file changed, 11 insertions(+), 135 deletions(-) diff --git a/README.md b/README.md index 2222d0c..4f73963 100644 --- a/README.md +++ b/README.md @@ -48,20 +48,12 @@ git cirrus build For more detailed docs see the [Installation Docs](https://github.com/evansde77/cirrus/wiki/Installation) -User Configuration File: -======================== - -As cirrus works as a git extension, it will use your gitconfig file. The installer will create a cirrus section in this file and create the following settings: - -1. *github-user* - Your Github Username -1. *github-token* - A github access token - Package Configuration Files: ============================ The per package controls used by cirrus live in a cirrus.conf file in the top level of the repo you use with cirrus. -This file, coupled with the cirrus setup.py template and command line tools dictate the behaviour of the cirrus commands within the package. Details for the cirrus config are in the (TBA) Configuration.MD file +This file, coupled with the cirrus setup.py template and command line tools dictate the behaviour of the cirrus commands within the package. Details for the cirrus config are in the [Cirrus Configuration Docs](https://github.com/evansde77/cirrus/wiki/CirrusConfiguration) Cirrus Commands: @@ -69,134 +61,18 @@ Cirrus Commands: See the [Cirrus Commands Docs](https://github.com/evansde77/cirrus/wiki#command-reference) -#### cirrus hello -A simple test command that says hello, verifies that things are working and prints out some info about your cirrus install - -Usage: -```bash -git cirrus hello -``` - - -* [git cirrus build](https://github.com/evansde77/cirrus/wiki/BuildCommand) -* [git cirrus test](https://github.com/evansde77/cirrus/wiki/TestCommand) -* [git cirrus release](https://github.com/evansde77/cirrus/wiki/ReleaseCommand) -* [git cirrus feature](https://github.com/evansde77/cirrus/wiki/FeatureCommand) -* [git cirrus docker-image](https://github.com/evansde77/cirrus/wiki/DockerImageCommand) -* [git cirrus selfupdate](https://github.com/evansde77/cirrus/wiki/SelfupdateCommand) - - - - -#### cirrus review -The cirrus review command provides some utilities for dealing with GitHub pull requests from the cirrus command line. -Available commands are: - - * git cirrus review list - list all open PRs for the repo, accepts -u or --user to filter for requests from a specific user - * git cirrus review details - Get details for a specific PR, specified by --id - * git cirrus review plusone - Set a Github context for the PR to indicate that the PR has been approved - * git cirrus review review - Add a review comment to a PR, optionally adding the plusone flag to it as well - - -Examples: - -```bash -git cirrus review list --user evansde77 # list open PRs by user evansde77 -git cirrus review list # list all open PRs -git cirrus review plusone --id 500 -c "+1" # adds the +1 context to the feature via a status update and sets it to success -git cirrus review reviee --id 500 -m "great work, LGTM" --plus-one -c "+1" # adds a comment to the PR and sets the +1 context status to success -``` - - - - - - -#### cirrus qc -Command for running quality control checks via pylint, pyflakes, pep8. - -Usage: -```bash -git cirrus qc --files --only-changes --pylint --pyflakes --pep8 -``` - -Options and config: -Running with no arguments will run all checks on everything. To run only a specific checker (pylint, pyflakes, or pep8) use the corosponding argument or a combonation of them. -Specific files may be ran using '--files' OR check only files that have not yet been commited to the repo by using the '--only-changes' argument. -For pylint, a score threshold must be set in cirrus.conf [quality] threshold. The path to an optional rcfile (pylint configuration) may be set at [quality] rcfile. +* [git cirrus hello](https://github.com/evansde77/cirrus/wiki/HelloCommand) - Install check and version info +* [git cirrus build](https://github.com/evansde77/cirrus/wiki/BuildCommand) - Create a development environment +* [git cirrus test](https://github.com/evansde77/cirrus/wiki/TestCommand) - Run test suites +* [git cirrus release](https://github.com/evansde77/cirrus/wiki/ReleaseCommand) - Release code and push to pypi +* [git cirrus feature](https://github.com/evansde77/cirrus/wiki/FeatureCommand) - Work on new features +* [git cirrus docker-image](https://github.com/evansde77/cirrus/wiki/DockerImageCommand) - Build and release container images +* [git cirrus selfupdate](https://github.com/evansde77/cirrus/wiki/SelfupdateCommand) - Update cirrus +* [git cirrus qc](https://github.com/evansde77/cirrus/wiki/QCCommand) - Run quality control and code standard tests +* [git cirrus docs](https://github.com/evansde77/cirrus/wiki/DocsCommand) - Build sphinx package docs +* [git cirrus review](https://github.com/evansde77/cirrus/wiki/ReviewCommand) - Helper for GitHub Pull Requests -#### cirrus docs -Command for publishing Sphinx documentation -Usage: -```bash -git cirrus docs build -git cirrus docs pack -git cirrus docs publish -``` - -Options and config: - -1. `git cirrus docs build`: `--make ` - 1. When run without `--make`, the default options `clean html` are used - 2. Requires a `sphinx_makefile_dir` value set in the `doc` section of cirrus.conf. - 3. `sphinx_makefile_dir` should point to the directory that contains Sphinx's Makefile. -2. `git cirrus docs pack` requires the following options in cirrus.conf [doc] section: - * sphinx_doc_dir - should point to the directory where the documentation to be packaged is. - E.g. /docs/\_build/html - * artifact_dir - should point to the directory where the documentation artifact should be saved. -3. `git cirrus docs publish` requires the following options in cirrus.conf [doc] section: - * publisher - the publisher plugin to use - 1. The publisher selected should have a section in cirrus.conf which contains the publisher options. Available publisher plugins can be found in /cirrus/plugins/publishers - 1. If using the `doc_file_server` plugin: - 1. in cirrus.conf: - * doc_file_server_url - the URL of the server the documentation is uploaded to - * doc_file_server_upload_path - the path to the location on the server the documentation should be uploaded to - * doc_file_server_sudo - a value of True or False for if sudo should be used when issuing the Fabric `put` command - _Note:_ Optional if doc_file_server_sudo is False - 2. in the [cirrus] section of your .gitconfig: - * file-server-username - the username used for the documentation file server - * file-server-keyfile - the path the ssh keyfile to use when uploading the documentation - 2. If using the `jenkins` plugin: - 1. in cirrus.conf: - * url - the URL of the Jenkins server - * doc_job - the name of the Jenkins job for the documentation build - * doc_var - the variable name which the uploaded documentation tarball will be accessed by (Jenkins File Parameter) - * arc_var - the variable that will be used to name the file/folder the archive should be unpacked to as determined by the name of the archive filename. I.e. package-0.0.0.tar.gz => package-0.0.0 (Jenkins String Parameter) - * extra_vars - boolean value indicating if there are move variables to send to Jenkins which should be defined in the section [jenkins_docs_extra_vars] - 2. in the [cirrus] section of your .gitconfig: - * buildserver-user - Jenkins username for authorization - * buildserver-token - token or password for authorization - -Example cirrus.conf: - -```ini -[doc] -sphinx_makefile_dir = docs/ -sphinx_doc_dir = docs/_build/html -artifact_dir = docs/artifacts -publisher = doc_file_server - -[doc_file_server] -doc_file_server_url = http://localhost:8080 -doc_file_server_upload_path = /docs/package/archive -doc_file_server_sudo = True -``` - -If using `publisher = jenkins`: - -```ini -[jenkins] -url = https://localhost:8080 -doc_job = doc_build -doc_var = artifact -arc_var = ARCHIVE -extra_vars = True - -[jenkins_docs_extra_vars] -var = value -var1 = value1 -``` From f0612472d98c774d47dd6db59b77ef7370601b5f Mon Sep 17 00:00:00 2001 From: devans Date: Thu, 31 Aug 2017 10:37:18 -0500 Subject: [PATCH 4/9] clean newlines --- src/cirrus/package_container.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/cirrus/package_container.py b/src/cirrus/package_container.py index f84628f..91c36c8 100644 --- a/src/cirrus/package_container.py +++ b/src/cirrus/package_container.py @@ -61,8 +61,7 @@ """ LOCAL_INSTALL_SCRIPT = \ -""" -#!/bin/bash +"""#!/bin/bash {virtualenv} pip install /opt/{{{{cirrus.configuration.package.name}}}}-{{{{cirrus.configuration.package.version}}}}.tar.gz @@ -70,14 +69,14 @@ """ PYPI_INSTALL_SCRIPT = \ -""" -#!/bin/bash +"""#!/bin/bash {virtualenv} pip install {pip_options} {{{{cirrus.configuration.package.name}}}}=={{{{cirrus.configuration.package.version}}}} """ + def make_executable(path): mode = os.stat(path).st_mode mode |= (mode & 0o444) >> 2 # copy R bits to X From 232146eaa5a4f7257380e102e493835bf916480d Mon Sep 17 00:00:00 2001 From: devans Date: Thu, 31 Aug 2017 14:34:04 -0500 Subject: [PATCH 5/9] add beta nightly release support --- src/cirrus/github_tools.py | 12 +++ src/cirrus/package_container.py | 19 +++-- src/cirrus/release.py | 36 +++++--- src/cirrus/release_utils.py | 88 ++++++++++++++++++++ tests/unit/cirrus/package_container_tests.py | 12 ++- tests/unit/cirrus/release_test.py | 44 ++++++++++ 6 files changed, 191 insertions(+), 20 deletions(-) create mode 100644 src/cirrus/release_utils.py diff --git a/src/cirrus/github_tools.py b/src/cirrus/github_tools.py index 8494323..fa4e7aa 100644 --- a/src/cirrus/github_tools.py +++ b/src/cirrus/github_tools.py @@ -92,6 +92,18 @@ def branch_status_list(self, branch): for d in data: yield d + def commit_files_optional_push(self, commit_msg, push=True, *filenames): + """ + commit files to the repo, push remote if required. + + """ + self.repo.index.add(filenames) + # commits with message + self.repo.index.commit(commit_msg) + # push branch to origin + if push: + return self.repo.remotes.origin.push(self.repo.head) + def log_branch_status(self, branch): """ _log_branch_status_ diff --git a/src/cirrus/package_container.py b/src/cirrus/package_container.py index 91c36c8..b0f1a9e 100644 --- a/src/cirrus/package_container.py +++ b/src/cirrus/package_container.py @@ -8,6 +8,7 @@ """ import sys import os +import git import json from cirrus.logger import get_logger @@ -77,10 +78,13 @@ """ -def make_executable(path): +def make_executable(path, repo): mode = os.stat(path).st_mode mode |= (mode & 0o444) >> 2 # copy R bits to X os.chmod(path, mode) + r = git.Repo(repo) + r.git.update_index(path, chmod='+x') + def write_basic_dockerfile(opts, config, path): @@ -122,15 +126,15 @@ def write_json_file(path, data): json.dump(data, handle) -def write_script(path, content, **extras): +def write_script(repo, path,content, **extras): """write script content to a file""" LOGGER.info("writing script {}".format(path)) script = content.format(**extras) - with open(path, 'w') as handle: + with open(path, 'wb') as handle: handle.write(script) # run chmod +x on new script - make_executable(path) + make_executable(path, repo) def edit_cirrus_conf(opts, config): @@ -199,13 +203,15 @@ def init_container(opts): "excludes": ["post_script.sh", "post_script.sh", ".dockerstache"] }) write_json_file(context, {}) - write_script(pre_script, DOCKER_PRE_SCRIPT) + write_script(opts.repo, pre_script, DOCKER_PRE_SCRIPT) write_script( + opts.repo, local_install, LOCAL_INSTALL_SCRIPT, virtualenv=venv_option ) write_script( + opts.repo, pypi_install, PYPI_INSTALL_SCRIPT, virtualenv=venv_option, @@ -213,12 +219,13 @@ def init_container(opts): ) if opts.local_install: write_script( + opts.repo, post_script, DOCKER_POST_SCRIPT, copy_dist=LOCAL_INSTALL_COMMAND.format(package=config.package_name()) ) else: - write_script(post_script, DOCKER_POST_SCRIPT, copy_dist="") + write_script(opts.repo, post_script, DOCKER_POST_SCRIPT, copy_dist="") edit_cirrus_conf(opts, config) modified = [ diff --git a/src/cirrus/release.py b/src/cirrus/release.py index 715b28c..65ed7ad 100644 --- a/src/cirrus/release.py +++ b/src/cirrus/release.py @@ -28,6 +28,7 @@ from cirrus.plugins.jenkins import JenkinsClient from cirrus.req_utils import bump_package from cirrus.release_status import release_status +import cirrus.release_utils as rel_utils LOGGER = get_logger() @@ -256,6 +257,12 @@ def build_parser(argslist): action='store_true', dest='major' ) + new_command.add_argument( + '--nightly', + action='store_true', + dest='nightly', + default=False + ) new_command.add_argument( '--bump', nargs=2, @@ -466,20 +473,22 @@ def new_release(opts): """ LOGGER.info("Creating new release...") - if not highlander([opts.major, opts.minor, opts.micro]): - msg = "Can only specify one of --major, --minor or --micro" - LOGGER.error(msg) - raise RuntimeError(msg) - - fields = ['major', 'minor', 'micro'] - mask = [opts.major, opts.minor, opts.micro] - field = [x for x in itertools.compress(fields, mask)][0] - config = load_configuration() + if opts.nightly: + msg = "creating new nightly release..." + new_version = rel_utils.new_nightly() + field = 'nightly' + else: + if not highlander([opts.major, opts.minor, opts.micro]): + msg = "Can only specify one of --major, --minor or --micro" + LOGGER.error(msg) + raise RuntimeError(msg) - # version bump: - current_version = config.package_version() - new_version = bump_version_field(current_version, field) + fields = ['major', 'minor', 'micro'] + mask = [opts.major, opts.minor, opts.micro] + field = [x for x in itertools.compress(fields, mask)][0] + current_version = config.package_version() + new_version = bump_version_field(current_version, field) # release branch branch_name = "{0}{1}".format( @@ -786,6 +795,8 @@ def merge_release(opts): if not opts.skip_develop: ghc.pull_branch(develop, remote=not opts.no_remote) ghc.merge_branch(release_branch) + if rel_utils.is_nightly(tag): + rel_utils.remove_nightly(ghc) sha = ghc.repo.head.ref.commit.hexsha if rel_conf['wait_on_ci_develop']: @@ -864,6 +875,7 @@ def main(): opts = build_parser(sys.argv) if opts.command == 'new': new_release(opts) + if opts.command == 'new-version': make_new_version(opts) diff --git a/src/cirrus/release_utils.py b/src/cirrus/release_utils.py new file mode 100644 index 0000000..b73d51e --- /dev/null +++ b/src/cirrus/release_utils.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +import re +import datetime + +from cirrus.configuration import load_configuration + +DEFAULT_FORMAT = "%Y%m%d" + + +def nightly_config(conf=None): + """ + get the nightly config settings from + the release section + + """ + result = { + "nightly_format": DEFAULT_FORMAT, + "nightly_separator": "-nightly-" + } + if not conf: + conf = load_configuration() + if not conf.has_section('release'): + return result + result['nightly_format'] = conf.get_param("release", "nightly_format", result['nightly_format']) + result['nightly_separator'] = conf.get_param("release", "nightly_separator", result['nightly_separator']) + return result + + +def is_nightly(conf, version): + """ + return True/False if the version string + provided matches a nightly format. + + """ + reg = "^[0-9]+\.[0-9]+\.[0-9]+{}".format(conf['nightly_separator']) + matcher = re.compile(reg) + elems = matcher.split(version, 1) + if len(elems) == 2: + return True + return False + + +def new_nightly(): + """ + generate a new nightly version + + """ + cirrus_conf = load_configuration() + nightly_conf = nightly_config(cirrus_conf) + now = datetime.datetime.now() + ts = now.strftime(nightly_conf['nightly_format']) + current = cirrus_conf.package_version() + + nightly = "{version}{sep}{ts}".format( + version=current, + sep=nightly_conf['nightly_separator'], + ts=ts + ) + return nightly + + +def remove_nightly(ghc): + """ + remove the nightly part from the cirrus.conf version + """ + cirrus_conf = load_configuration() + nightly_conf = nightly_config(cirrus_conf) + current = cirrus_conf.package_version() + if is_nightly(nightly_conf, current): + new_version = current.split(nightly_conf['nightly_separator'], 1)[0] + cirrus_conf.update_package_version(new_version) + ghc.commit_files_optional_push( + "remove nightly tag from cirrus.conf", + False, + "cirrus.conf" + ) + return + + +if __name__ == '__main__': + c = nightly_config() + + + print is_nightly(c, '0.0.0-nightly-20170818') + print is_nightly(c, '0.0.0') + print new_nightly() + diff --git a/tests/unit/cirrus/package_container_tests.py b/tests/unit/cirrus/package_container_tests.py index 25a5fb2..24e9369 100644 --- a/tests/unit/cirrus/package_container_tests.py +++ b/tests/unit/cirrus/package_container_tests.py @@ -58,9 +58,13 @@ def tearDown(self): if os.path.exists(self.dir): os.system('rm -rf {}'.format(self.dir)) - def test_init_container(self): + @mock.patch('cirrus.package_container.git') + def test_init_container(self, mock_git): """test init_container call""" - + mock_repo = mock.Mock() + mock_repo.git = mock.Mock() + mock_repo.git.update_index = mock.Mock() + mock_git.Repo = mock.Mock(return_value=mock_repo) opts = mock.Mock() opts.repo = self.dir opts.template_dir = "container-template" @@ -86,6 +90,10 @@ def test_init_container(self): found = os.listdir(templates) for exp in expected_files: self.failUnless(exp in found) + self.failUnless(mock_repo.git.update_index.called) + for call in mock_repo.git.update_index.call_args_list: + self.failUnless(os.path.basename(call[0][0]) in expected_files) + self.assertEqual(call[1], {'chmod': '+x'}) dockerfile = os.path.join(templates, 'Dockerfile.mustache') with open(dockerfile, 'r') as handle: diff --git a/tests/unit/cirrus/release_test.py b/tests/unit/cirrus/release_test.py index 5abdac9..04eccb4 100644 --- a/tests/unit/cirrus/release_test.py +++ b/tests/unit/cirrus/release_test.py @@ -36,6 +36,8 @@ def setUp(self): ) self.harness = CirrusConfigurationHarness('cirrus.release.load_configuration', self.config) self.harness.setUp() + self.harness_utils = CirrusConfigurationHarness('cirrus.release_utils.load_configuration', self.config) + self.harness_utils.setUp() self.patch_pull = mock.patch('cirrus.release.checkout_and_pull') self.patch_branch = mock.patch('cirrus.release.branch') self.patch_commit = mock.patch('cirrus.release.commit_files_optional_push') @@ -48,6 +50,7 @@ def tearDown(self): self.patch_branch.stop() self.patch_commit.stop() self.harness.tearDown() + self.harness_utils.tearDown() if os.path.exists(self.dir): os.system('rm -rf {0}'.format(self.dir)) @@ -62,6 +65,7 @@ def test_new_release(self, mock_unstaged): opts.micro = True opts.major = False opts.minor = False + opts.nightly = False opts.bump = None # should create a new minor release, editing @@ -81,6 +85,44 @@ def test_new_release(self, mock_unstaged): self.assertEqual(self.mock_commit.call_args[0][2], False) self.assertEqual(self.mock_commit.call_args[0][3], 'cirrus.conf') + @mock.patch('cirrus.release.has_unstaged_changes') + @mock.patch('cirrus.release_utils.datetime') + def test_new_nightly_release(self, mock_dt, mock_unstaged): + """ + _test_new_release_ + + """ + mock_ts = mock.Mock() + mock_ts.strftime = mock.Mock(return_value="TIMESTAMP") + mock_now = mock.Mock(return_value=mock_ts) + mock_dt.datetime=mock.Mock() + mock_dt.datetime.now = mock_now + mock_unstaged.return_value = False + opts = mock.Mock() + opts.micro = False + opts.major = False + opts.minor = False + opts.nightly = True + opts.bump = None + + # should create a new minor release, editing + # the cirrus config in the test dir + new_release(opts) + + # verify new version + new_conf = Configuration(self.config) + new_conf.load() + self.assertEqual(new_conf.package_version(), '1.2.3-nightly-TIMESTAMP') + + self.failUnless(self.mock_pull.called) + self.assertEqual(self.mock_pull.call_args[0][1], 'develop') + self.failUnless(self.mock_branch.called) + self.assertEqual(self.mock_branch.call_args[0][1], 'release/1.2.3-nightly-TIMESTAMP') + self.failUnless(self.mock_commit.called) + self.assertEqual(self.mock_commit.call_args[0][2], False) + self.assertEqual(self.mock_commit.call_args[0][3], 'cirrus.conf') + + @mock.patch('cirrus.release.has_unstaged_changes') @mock.patch('cirrus.release.bump_package') def test_new_release_bump(self, mock_bump, mock_unstaged): @@ -93,6 +135,7 @@ def test_new_release_bump(self, mock_bump, mock_unstaged): opts.micro = True opts.major = False opts.minor = False + opts.nightly = False opts.bump = [['womp', '1.2.3'], ['wibble', '3.4.5']] # should create a new minor release, editing @@ -125,6 +168,7 @@ def test_new_release_unstaged(self, mock_unstaged): opts.micro = True opts.major = False opts.minor = False + opts.nightly = False opts.bump = None self.assertRaises(RuntimeError, new_release, opts) From 22032dbd8652a59d4a631283aa6bedc858f70710 Mon Sep 17 00:00:00 2001 From: devans Date: Thu, 31 Aug 2017 15:04:38 -0500 Subject: [PATCH 6/9] tweaks from testing --- src/cirrus/release.py | 2 +- src/cirrus/release_utils.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cirrus/release.py b/src/cirrus/release.py index 65ed7ad..9dbc9b5 100644 --- a/src/cirrus/release.py +++ b/src/cirrus/release.py @@ -474,6 +474,7 @@ def new_release(opts): """ LOGGER.info("Creating new release...") config = load_configuration() + current_version = config.package_version() if opts.nightly: msg = "creating new nightly release..." new_version = rel_utils.new_nightly() @@ -487,7 +488,6 @@ def new_release(opts): fields = ['major', 'minor', 'micro'] mask = [opts.major, opts.minor, opts.micro] field = [x for x in itertools.compress(fields, mask)][0] - current_version = config.package_version() new_version = bump_version_field(current_version, field) # release branch diff --git a/src/cirrus/release_utils.py b/src/cirrus/release_utils.py index b73d51e..da735de 100644 --- a/src/cirrus/release_utils.py +++ b/src/cirrus/release_utils.py @@ -27,12 +27,13 @@ def nightly_config(conf=None): return result -def is_nightly(conf, version): +def is_nightly(version): """ return True/False if the version string provided matches a nightly format. """ + conf = nightly_config() reg = "^[0-9]+\.[0-9]+\.[0-9]+{}".format(conf['nightly_separator']) matcher = re.compile(reg) elems = matcher.split(version, 1) @@ -67,7 +68,7 @@ def remove_nightly(ghc): cirrus_conf = load_configuration() nightly_conf = nightly_config(cirrus_conf) current = cirrus_conf.package_version() - if is_nightly(nightly_conf, current): + if is_nightly(current): new_version = current.split(nightly_conf['nightly_separator'], 1)[0] cirrus_conf.update_package_version(new_version) ghc.commit_files_optional_push( From ac63a565f16b51781f26f303d2ef1d60e2942cba Mon Sep 17 00:00:00 2001 From: devans Date: Thu, 31 Aug 2017 15:16:27 -0500 Subject: [PATCH 7/9] tweaks for py3 --- src/cirrus/package_container.py | 4 ++-- src/cirrus/release_utils.py | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/cirrus/package_container.py b/src/cirrus/package_container.py index b0f1a9e..8ff2dca 100644 --- a/src/cirrus/package_container.py +++ b/src/cirrus/package_container.py @@ -126,12 +126,12 @@ def write_json_file(path, data): json.dump(data, handle) -def write_script(repo, path,content, **extras): +def write_script(repo, path, content, **extras): """write script content to a file""" LOGGER.info("writing script {}".format(path)) script = content.format(**extras) - with open(path, 'wb') as handle: + with open(path, 'w') as handle: handle.write(script) # run chmod +x on new script make_executable(path, repo) diff --git a/src/cirrus/release_utils.py b/src/cirrus/release_utils.py index da735de..0dec8da 100644 --- a/src/cirrus/release_utils.py +++ b/src/cirrus/release_utils.py @@ -77,13 +77,3 @@ def remove_nightly(ghc): "cirrus.conf" ) return - - -if __name__ == '__main__': - c = nightly_config() - - - print is_nightly(c, '0.0.0-nightly-20170818') - print is_nightly(c, '0.0.0') - print new_nightly() - From 46ff3cc0275274ba46d60adf1ad8425ed96f7906 Mon Sep 17 00:00:00 2001 From: devans Date: Thu, 31 Aug 2017 15:28:28 -0500 Subject: [PATCH 8/9] fix for #174 --- src/cirrus/plugins/builders/conda_env.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/cirrus/plugins/builders/conda_env.py b/src/cirrus/plugins/builders/conda_env.py index b4f303e..3dafbd6 100644 --- a/src/cirrus/plugins/builders/conda_env.py +++ b/src/cirrus/plugins/builders/conda_env.py @@ -29,10 +29,9 @@ def create(self, **kwargs): conda = kwargs.get('conda', self.conda_bin) upgrade = kwargs.get('upgrade', False) nosetupdevelop = kwargs.get('nosetupdevelop', False) - environment = kwargs.get( - 'environment', - self.build_config.get('conda-environment', None) - ) + environment = kwargs.get('environment', None) + if environment is None: + environment = self.build_config.get('conda-environment', None) if environment is None: msg = "No conda environment yaml specified in cirrus.conf [build] section or via --environment option" LOGGER.error(msg) From 269a04fcdc3c8bd41f2970d88cf4be6ba78ff521 Mon Sep 17 00:00:00 2001 From: devans Date: Thu, 31 Aug 2017 16:31:24 -0500 Subject: [PATCH 9/9] cirrus release: new release created for release/0.2.7 --- cirrus.conf | 2 +- src/cirrus/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cirrus.conf b/cirrus.conf index dfc8cc6..70169d9 100644 --- a/cirrus.conf +++ b/cirrus.conf @@ -1,6 +1,6 @@ [package] name = cirrus-cli -version = 0.2.6 +version = 0.2.7 description = cirrus development and build git extensions organization = evansde77 version_file = src/cirrus/__init__.py diff --git a/src/cirrus/__init__.py b/src/cirrus/__init__.py index dc51663..64ddd7d 100644 --- a/src/cirrus/__init__.py +++ b/src/cirrus/__init__.py @@ -18,5 +18,5 @@ See the License for the specific language governing permissions and limitations under the License. """ -__version__ = "0.2.6" +__version__ = "0.2.7"