diff --git a/README.md b/README.md index b944622..3ad4ed9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +:warning: cirrus has been ported to Python 3 and newer versions (>=3.0.0) are not backwards compatible. The last Python 2 release was 2.0.2. :warning: + cirrus ====== @@ -18,7 +20,7 @@ Our solution was to check out the most recently working version, `0.1.7` and cre Installation Prerequisites ========================== -* Cirrus requires python 2.7 (support for python 3 is in the pipeline) as well as pip and virtualenv installed. +* Cirrus requires python 3.5. * Git tools are heavily used, git is a requirement as cirrus is accessed via git command aliases. Installation as a user: @@ -31,7 +33,7 @@ bash installer.sh The installer script will set up an install of cirrus for you in your home directory and prompt for some info so that it can set up some parameters in your .gitconfig -The installer will create a virtualenv and install cirrus from pip via the cirrus-cli package, installing the latest available version. +The installer will create a virtualenv and install cirrus from pip via the cirrus-cli package, installing the latest available version. If a specific version is required, it may be set via environment variable. For example, if you require version 3.0.0 you can use `export CIRRUS_INSTALL_VERSION='==3.0.0'` @@ -42,7 +44,7 @@ _Note_: This package uses GitFlow, any development work should be done off the d pull requests made against develop, not master. ```bash -git clone https://github.com/evansde77/cirrus.git +git clone https://github.com/cloudant/cirrus.git cd cirrus git cirrus build ``` @@ -197,9 +199,8 @@ Commands related to creation of a new git-flow style release branch, building th There are three subcommands: 1. new - creates a new release branch, increments the package version, builds the release notes if configured. -2. build - Runs sdist to create a new build artifact from the release branch +2. build\_and\_upload - builds a release artifact and uploads it to artifactory. A `--dev` option is provided to create test releases which can be used for testing (via sapitest002 for example). 3. merge - Runs git-flow style branch merges back to master and develop, optionally waiting on CI or setting flags for GH build contexts if needed -4. upload - Pushes the build artifact to the pypi server configured in the cirrus conf, using a plugin system to allow for customisation. Usage: ```bash diff --git a/cirrus.conf b/cirrus.conf index e2e6457..546bc01 100644 --- a/cirrus.conf +++ b/cirrus.conf @@ -1,6 +1,6 @@ [package] name = cirrus-cli -version = 3.0.0 +version = 3.1.0 description = cirrus development and build git extensions organization = cloudant version_file = src/cirrus/__init__.py diff --git a/installer.sh b/installer.sh index 64e04e5..4b15020 100644 --- a/installer.sh +++ b/installer.sh @@ -34,7 +34,7 @@ echo "Installing cirrus to LOCATION=${LOCATION}" > ${LOCATION}/install.log cd ${LOCATION} # bootstrap virtualenv -virtualenv venv +python -m venv venv . venv/bin/activate # This depends on a properly configured pip.conf file. diff --git a/src/cirrus/__init__.py b/src/cirrus/__init__.py index 94fb774..5195a4b 100644 --- a/src/cirrus/__init__.py +++ b/src/cirrus/__init__.py @@ -18,5 +18,5 @@ See the License for the specific language governing permissions and limitations under the License. """ -__version__="3.0.0" +__version__="3.1.0" diff --git a/src/cirrus/build.py b/src/cirrus/build.py index bed38e6..b8b2f0f 100644 --- a/src/cirrus/build.py +++ b/src/cirrus/build.py @@ -74,6 +74,11 @@ def build_parser(argslist): default=False, action='store_true' ) + parser.add_argument( + '--freeze', + action='store_true', + help='ignores sub-dependency pins, and then updates them' + ) opts = parser.parse_args(argslist) return opts @@ -99,6 +104,14 @@ def execute_build(opts): # we have custom build controls in the cirrus.conf reqs_name = build_params.get('requirements_file', 'requirements.txt') + activate_cmd = '. ./{}/bin/activate'.format(config.venv_name()) + if opts.freeze: + requirements = Requirements( + reqs_name, + 'requirements-sub.txt', + activate_cmd + ) + requirements.parse_out_sub_dependencies() extra_reqs = build_params.get('extra_requirements', '') extra_reqs = [x.strip() for x in extra_reqs.split(',') if x.strip()] if opts.extras: @@ -214,11 +227,15 @@ def execute_build(opts): else: LOGGER.info('running python setup.py develop...') run( - '. ./{0}/bin/activate && python setup.py develop'.format( - config.venv_name() + '{} && python setup.py develop'.format( + activate_cmd ) ) + if opts.freeze: + requirements.update_sub_dependencies() + requirements.restore_sub_dependencies() + def main(): """ @@ -233,5 +250,65 @@ def main(): build_docs(make_opts=opts.docs) +class Requirements: + """ + Modify requirments files + """ + def __init__(self, dep_filename, sub_dep_filename, activate_cmd): + """ + :param str dep_filename: name of the requirements file containing + direct dependencies + :param str sub_dep_filename: name of the requirements file containing + sub-dependencies + :param str activate_cmd: shell command used to activate a virtualenv + """ + self.direct_filename = dep_filename + self.sub_filename = sub_dep_filename + self.direct_dependencies = None + self.sub_dependencies = None + self.activate_cmd = activate_cmd + + def parse_out_sub_dependencies(self): + """ + Modifies the direct requirements file to exclude sub-dependencies, + noop if no sub-dependencies specified + """ + with open(self.direct_filename, 'r') as requirements_file: + direct = requirements_file.read().strip().split('\n') + del direct[0] + with open(self.direct_filename, 'w+') as new_req_file: + direct = sorted(direct) + new_req_file.write('\n'.join(direct)) + + self.direct_dependencies = set(direct) + + def update_sub_dependencies(self): + """ + Re-pins sub-dependencies after getting the latest acceptable versions + """ + run('{} && pip freeze > {}'.format( + self.activate_cmd, self.sub_filename + )) + with open(self.sub_filename, 'r') as sub_req_file: + sub = sub_req_file.read().strip().split('\n') + sub = [item for item in sub if not item.startswith('-e')] + self.sub_dependencies = set(sub) + new_sub_req = self.sub_dependencies - self.direct_dependencies + with open(self.sub_filename, 'w+') as sub_req_file: + new_sub_req = sorted(list(new_sub_req)) + sub_req_file.write('\n'.join(new_sub_req)) + + def restore_sub_dependencies(self): + """ + Modifies requirments file to include sub-dependencies + """ + with open(self.direct_filename, 'r+') as requirements_file: + contents = requirements_file.read() + requirements_file.seek(0) + requirements_file.write( + '-r {}\n'.format(self.sub_filename) + contents + ) + + if __name__ == '__main__': main() diff --git a/src/cirrus/selfupdate.py b/src/cirrus/selfupdate.py index bcc6382..d23e429 100644 --- a/src/cirrus/selfupdate.py +++ b/src/cirrus/selfupdate.py @@ -11,7 +11,6 @@ import argparse import arrow import os -import requests import inspect import contextlib @@ -26,7 +25,6 @@ LOGGER = get_logger() -PYPI_JSON_URL = "https://pypi.python.org/pypi/cirrus-cli/json" @contextlib.contextmanager @@ -58,7 +56,7 @@ def build_parser(argslist): '--version', help='specify a tag to install', required=False, - default=None, + default='', ) parser.add_argument( @@ -80,40 +78,6 @@ def build_parser(argslist): return opts -def sort_by_date(d1, d2): - """ - cmp function to sort by datetime string - that is second element of tuples in list - """ - date1 = arrow.get(d1[1]) - date2 = arrow.get(d2[1]) - return date1 > date2 - - -def latest_release(config): - """ - _latest_release_ - - pull list of releases from GH repo, pick the newest by - publication date. - - """ - releases = get_releases(config.organisation_name(), config.package_name()) - tags = [(release['tag_name'], release['published_at']) for release in releases] - sorted(tags, cmp=sort_by_date) - most_recent_tag = tags[0][0] - return most_recent_tag - - -def latest_pypi_release(): - """grab latest release from pypi""" - resp = requests.get(PYPI_JSON_URL) - resp.raise_for_status() - content = resp.json() - latest = content['info']['version'] - return latest - - def find_cirrus_install(): """ _find_cirrus_install_ @@ -153,7 +117,9 @@ def setup_develop(config): def pip_install(version): """pip install the version of cirrus requested""" - pip_req = 'cirrus-cli=={0}'.format(version) + if version: + version = '=={}'.format(version) + pip_req = 'cirrus-cli{}'.format(version) venv_name = os.path.basename(virtualenv_home()) LOGGER.info("running pip upgrade...") run( @@ -191,24 +157,14 @@ def pip_update(opts): """update pip installed cirrus""" install = cirrus_home() with chdir(install): - if opts.version is not None: - tag = opts.version - LOGGER.info("tag specified: {0}".format(tag)) - else: - # should probably be a pip call now... - tag = latest_pypi_release() - LOGGER.info("Retrieved latest tag: {0}".format(tag)) - pip_install(tag) + pip_install(opts.version) def main(): """ - _main_ - - parse command line opts and deduce wether to check out - a branch or tag, default behaviour is to look up latest - release on github and install that - + Parses command line opts and deduce wether to check out + a branch or tag, default behaviour is to install latest release found by + pip """ opts = build_parser(sys.argv) if opts.legacy_repo: diff --git a/tests/unit/cirrus/build_tests.py b/tests/unit/cirrus/build_tests.py index 3a7e389..ed67db9 100644 --- a/tests/unit/cirrus/build_tests.py +++ b/tests/unit/cirrus/build_tests.py @@ -2,10 +2,11 @@ """ build command test coverage """ - +import os +import tempfile from unittest import TestCase, mock -from cirrus.build import execute_build, build_parser +from cirrus.build import Requirements, execute_build, build_parser class BuildParserTests(TestCase): @@ -100,6 +101,7 @@ def test_execute_build_default_pypi(self): opts.upgrade = False opts.extras = [] opts.nosetupdevelop = False + opts.freeze = False execute_build(opts) @@ -116,6 +118,7 @@ def test_execute_build_pypirc(self): opts.upgrade = False opts.extras = [] opts.nosetupdevelop = False + opts.freeze = False self.mock_conf.pypi_url.return_value = "dev" self.mock_pypirc_inst.index_servers = ['dev'] @@ -135,6 +138,7 @@ def test_execute_build_default_pypi_pip_options(self): opts.upgrade = False opts.extras = [] opts.nosetupdevelop = False + opts.freeze = False self.mock_conf.pip_options.return_value = "PIPOPTIONS" execute_build(opts) @@ -151,6 +155,7 @@ def test_execute_build_default_pypi_upgrade(self): opts.upgrade = True opts.extras = [] opts.nosetupdevelop = False + opts.freeze = False execute_build(opts) @@ -167,6 +172,7 @@ def test_execute_build_with_extras(self): opts.upgrade = False opts.extras = ['test-requirements.txt'] opts.nosetupdevelop = False + opts.freeze = False execute_build(opts) self.mock_local.assert_has_calls([ mock.call('CIRRUS_HOME/venv/bin/virtualenv CWD/venv'), @@ -182,6 +188,7 @@ def test_execute_build_with_nosetupdevelop(self): opts.extras = [] opts.upgrade = False opts.nosetupdevelop = True + opts.freeze = False execute_build(opts) self.mock_local.assert_has_calls([ @@ -196,6 +203,7 @@ def test_execute_build_with_pypi(self): opts.extras = [] opts.upgrade = False opts.nosetupdevelop = False + opts.freeze = False self.mock_conf.pypi_url.return_value = "PYPIURL" execute_build(opts) @@ -214,6 +222,7 @@ def test_execute_build_with_pypi_upgrade(self): opts.extras = [] opts.nosetupdevelop = False opts.upgrade = True + opts.freeze = False self.mock_conf.pypi_url.return_value = "PYPIURL" execute_build(opts) @@ -223,3 +232,106 @@ def test_execute_build_with_pypi_upgrade(self): mock.call('CWD/venv/bin/pip install -i https://PYPIUSERNAME:TOKEN@PYPIURL/simple --upgrade -r requirements.txt'), mock.call('. ./venv/bin/activate && python setup.py develop') ]) + + @mock.patch('cirrus.build.Requirements', spec=Requirements) + def test_execute_build_freeze(self, mock_requirements): + """test execute_build with default pypi settings""" + opts = mock.Mock() + opts.clean = False + opts.upgrade = False + opts.extras = [] + opts.nosetupdevelop = False + opts.freeze = True + m_requirements = mock_requirements.return_value + + execute_build(opts) + + self.mock_local.assert_has_calls([ + mock.call('CIRRUS_HOME/venv/bin/virtualenv CWD/venv'), + mock.call('CWD/venv/bin/pip install -r requirements.txt'), + mock.call('. ./venv/bin/activate && python setup.py develop') + ]) + mock_requirements.assert_called_once_with( + 'requirements.txt', + 'requirements-sub.txt', + '. ./venv/bin/activate' + ) + self.assertTrue(m_requirements.parse_out_sub_dependencies.called) + self.assertTrue(m_requirements.update_sub_dependencies.called) + self.assertTrue(m_requirements.restore_sub_dependencies.called) + + +class RequirementsTests(TestCase): + @classmethod + def setUpClass(cls): + cls.requirements_file = tempfile.mkstemp(text=True)[1] + with open(cls.requirements_file, 'w+') as f: + f.write( + '-r test-requirements-sub.txt\n' + 'Doge==0.0.0\n' + 'bar==0.0.0\n' + 'carburetor==16.1.0.e321a26\n' + 'foo==0.0.0\n' + ) + cls.sub_requirements_file = tempfile.mkstemp(text=True)[1] + with open(cls.sub_requirements_file, 'w+') as f: + f.write( + 'baz==0.0.0\n' + 'oof==0.0.0\n' + ) + cls.requirements = Requirements( + cls.requirements_file, + cls.sub_requirements_file, + '. ./venv/bin/activate' + ) + + @classmethod + def tearDownClass(cls): + os.remove(cls.requirements_file) + os.remove(cls.sub_requirements_file) + + @mock.patch('cirrus.build.run') + def test_requirements(self, mock_run): + # Ensures the nested requirments file is removed + self.requirements.parse_out_sub_dependencies() + with open(self.requirements_file, 'r') as f: + req_contents = f.read() + self.assertEqual( + 'Doge==0.0.0\nbar==0.0.0\ncarburetor==16.1.0.e321a26\nfoo==0.0.0', + req_contents + ) + + # Ensures sub dependencies get updated + mock_run.side_effect = self.mock_pick_freeze + self.requirements.update_sub_dependencies() + + with open(self.sub_requirements_file, 'r') as f: + new_contents = f.read() + self.assertEqual('baz==0.0.1\noof==0.1.0', new_contents) + + # Ensures the nested requirements file is restored + self.requirements.restore_sub_dependencies() + with open(self.requirements_file, 'r') as f: + req_contents = f.read() + sub_required_line = '-r {}\n'.format(self.sub_requirements_file) + self.assertEqual( + '{}' + 'Doge==0.0.0\n' + 'bar==0.0.0\n' + 'carburetor==16.1.0.e321a26\n' + 'foo==0.0.0'.format(sub_required_line), + req_contents + ) + + def mock_pick_freeze(self, *args): + os.remove(self.sub_requirements_file) + with open(self.sub_requirements_file, 'w+') as f: + f.write( + '-e git+somegiturl' + 'Doge==0.0.0\n' + 'bar==0.0.0\n' + 'baz==0.0.1\n' + 'carburetor==16.1.0.e321a26\n' + 'foo==0.0.0\n' + 'oof==0.1.0' + )