From 04d2579ae97e21311076e619bf833b6a4bbc0eea Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Thu, 18 Jun 2020 14:57:34 +0300 Subject: [PATCH 1/8] added release management --- .circleci/config.yml | 188 +++++++++++++++++++++++++++++++++-- .circleci/package_release.py | 12 +++ 2 files changed, 190 insertions(+), 10 deletions(-) create mode 100644 .circleci/package_release.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 720e915..7043bcd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -version: 2 +version: 2.1 checkout: post: @@ -9,10 +9,22 @@ checkout: git checkout -qf FETCH_HEAD fi -jobs: - flake8: +executors: + py27: docker: - image: circleci/python:2.7 + + py36: + docker: + - image: circleci/python:3.6 + + wagon_generator: + machine: + image: ubuntu-1604:201903-01 + +commands: + + flake8: steps: - checkout - run: pip install flake8 --user @@ -20,9 +32,7 @@ jobs: name: Run flake8 command: flake8 fabric_plugin - unittests: - docker: - - image: circleci/python:2.7 + run_unittests: steps: - checkout - run: @@ -40,9 +50,167 @@ jobs: - store_test_results: path: test-results + check_py3_compat: + steps: + - run: + name: install futurize + command: pip install future --user + - run: + # those modules contain code that futurize would want to change, + # so let's just remove them so that it doesn't report on them + # (there's no "exclude" functionality in futurize) + name: remove compat modules + command: | + rm fabric_plugin/_compat.py + - run: + name: find python3-incompatible code + command: | + FUTURIZE="futurize ." + while read line; do + [[ "$line" =~ ^#.* ]] && continue + FUTURIZE="${FUTURIZE} ${line}" + done<.circleci/py3fixers + echo "Running: $FUTURIZE" + $FUTURIZE>futurize_diffs + - run: + name: check that there is no python3-incompatible code + command: | + if [[ -s futurize_diffs ]]; then + echo "Python-3-incompatible code found" + cat futurize_diffs + exit 1 + fi + generate_py27py36_wagon: + steps: + - run: + name: Create Workspace Build directory. + command: mkdir -p workspace/build + - run: + name: Build py27py36 Wagon + command: | + git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git + docker build -t cloudify-centos-7-wagon-builder cloudify-wagon-build-containers/centos_7_py2py3 + docker run -v ~/project/:/packaging cloudify-centos-7-wagon-builder + - run: + name: copy wagon to workspace + command: cp *.wgn workspace/build/ + - persist_to_workspace: + root: workspace + paths: + - build/* + + generate_rhel_py27py36_wagon: + steps: + - run: + name: Create Workspace Build directory. + command: mkdir -p workspace/build + - run: + name: Build RHEL py27py36 Wagon + command: | + git clone https://github.com/cloudify-cosmo/cloudify-wagon-build-containers.git + docker build -t cloudify-redhat-7-wagon-builder cloudify-wagon-build-containers/redhat_7_py2py3 --build-arg USERNAME="$USERNAME" --build-arg PASSWORD="$PASSWORD" + docker run -v ~/project/:/packaging cloudify-redhat-7-wagon-builder + - run: + name: copy wagon to workspace + command: cp *.wgn workspace/build/ + - persist_to_workspace: + root: workspace + paths: + - build/* + + release_plugin: + steps: + - run: + name: "Pull Submodules" + command: | + git submodule init + git submodule update --remote --recursive + - run: + name: Download pip + command: curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py" + - run: + name: Install pip + command: sudo python get-pip.py + - run: + name: Install virtualenv + command: pip install --user virtualenv + - run: + name: Init virtualenv + command: virtualenv env + - run: + name: install tox + command: pip install --user pygithub pyyaml==3.10 + - run: + name: upgrade setuptools + command: pip install --upgrade setuptools + - run: + name: install local project + command: pip install https://github.com/cloudify-incubator/cloudify-ecosystem-test/archive/latest.zip + - attach_workspace: + at: workspace + - run: python .circleci/package_release.py +jobs: + + unittests_py27: + executor: py27 + steps: + - checkout + - flake8 + - run_unittests + + unittests_py36: + executor: py36 + steps: + - checkout + - flake8 + - run_unittests + + py3_compat: + executor: py27 + steps: + - checkout + - check_py3_compat + + wagon: + executor: wagon_generator + steps: + - checkout + - generate_py27py36_wagon + # - generate_py3_wagon # TODO: When we have Python3 Manager Integration Test. + + rhel_wagon: + executor: wagon_generator + steps: + - checkout + - generate_rhel_py27py36_wagon + + release: + executor: py27 + steps: + - checkout + - release_plugin + workflows: - version: 2 + version: 2.1 tests: - jobs: - - flake8 - - unittests + jobs: &all_jobs + - py3_compat + - unittests_py27 + - unittests_py36 + - wagon: + filters: + branches: + only: /([0-9\.]*\-build|master|dev|replace-wagon-builder-to-py2py3-builder)/ + - rhel_wagon: + filters: + branches: + only: /([0-9\.]*\-build|master|dev|replace-wagon-builder-to-py2py3-builder)/ + - release: + filters: + branches: + only: /master/ + requires: + - wagon + - rhel_wagon + + diff --git a/.circleci/package_release.py b/.circleci/package_release.py new file mode 100644 index 0000000..9e6919b --- /dev/null +++ b/.circleci/package_release.py @@ -0,0 +1,12 @@ +from os import path, pardir +from ecosystem_cicd_tools.release import ( + plugin_release_with_latest, find_version) + +setup_py = path.join( + path.abspath(path.join(path.dirname(__file__), pardir)), + 'setup.py') + + +if __name__ == '__main__': + plugin_release_with_latest( + 'cloudify-fabric-plugin', find_version(setup_py)) \ No newline at end of file From 43626c2ead41083b7b479dfb82422b1a9809c4d3 Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Thu, 18 Jun 2020 14:58:41 +0300 Subject: [PATCH 2/8] disable rhel wagon job --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7043bcd..9d9f27d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -182,7 +182,7 @@ jobs: executor: wagon_generator steps: - checkout - - generate_rhel_py27py36_wagon +# - generate_rhel_py27py36_wagon release: executor: py27 From 73c75ea457153594e8f01983199b26dda8cca398 Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Thu, 18 Jun 2020 15:02:44 +0300 Subject: [PATCH 3/8] added py3fixers --- .circleci/py3fixers | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .circleci/py3fixers diff --git a/.circleci/py3fixers b/.circleci/py3fixers new file mode 100644 index 0000000..d455619 --- /dev/null +++ b/.circleci/py3fixers @@ -0,0 +1,21 @@ +--stage1 +-f lib2to3.fixes.fix_getcwdu +-f lib2to3.fixes.fix_long +-f lib2to3.fixes.fix_nonzero +-f lib2to3.fixes.fix_input +-f lib2to3.fixes.fix_raw_input +-f lib2to3.fixes.fix_itertools +-f lib2to3.fixes.fix_itertools_imports +-f lib2to3.fixes.fix_exec +-f lib2to3.fixes.fix_operator +-f libfuturize.fixes.fix_execfile +-f libpasteurize.fixes.fix_newstyle +-f lib2to3.fixes.fix_filter +-f lib2to3.fixes.fix_map +-f lib2to3.fixes.fix_zip +-f lib2to3.fixes.fix_xrange +-f lib2to3.fixes.fix_basestring +-f libfuturize.fixes.fix_cmp +-f libfuturize.fixes.fix_division_safe +-f lib2to3.fixes.fix_metaclass +-f libfuturize.fixes.fix_unicode_keep_u From f4ac1a4c8ad10c773a7664cb040f67199193d0d7 Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Thu, 18 Jun 2020 15:15:42 +0300 Subject: [PATCH 4/8] fixed _compat --- fabric_plugin/_compat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fabric_plugin/_compat.py b/fabric_plugin/_compat.py index 15f9057..df416bd 100644 --- a/fabric_plugin/_compat.py +++ b/fabric_plugin/_compat.py @@ -31,6 +31,7 @@ def exec_(code, globs): """) else: + import builtins from io import StringIO exec_ = getattr(builtins, 'exec') From d6d002806ec5e09034aadd8f05bea5b0831a56da Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Thu, 18 Jun 2020 16:15:17 +0300 Subject: [PATCH 5/8] fix unitests to pass on py3 --- fabric_plugin/tests/test_fabric_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fabric_plugin/tests/test_fabric_plugin.py b/fabric_plugin/tests/test_fabric_plugin.py index dc1c5b2..b176e6e 100644 --- a/fabric_plugin/tests/test_fabric_plugin.py +++ b/fabric_plugin/tests/test_fabric_plugin.py @@ -108,8 +108,8 @@ def test_missing_tasks_file(self): self._execute('test.run_task', tasks_file='missing.py') def test_bad_tasks_file(self): - with self.assertRaisesRegexp(NonRecoverableError, - "ImportError: No module named module"): + + with self.assertRaisesRegexp(NonRecoverableError, "No module named"): self._execute('test.run_task', tasks_file='corrupted_file.py') def test_missing_task(self): From b4474bc8f2669e7229488ff5e246764c8bd3d353 Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Thu, 18 Jun 2020 17:59:15 +0300 Subject: [PATCH 6/8] fixed dependencies --- dev-requirements.txt | 1 - plugin.yaml | 4 ++-- setup.cfg | 2 -- setup.py | 4 ++-- 4 files changed, 4 insertions(+), 7 deletions(-) delete mode 100644 setup.cfg diff --git a/dev-requirements.txt b/dev-requirements.txt index 8577a2a..e69de29 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1 +0,0 @@ -git+https://github.com/cloudify-cosmo/cloudify-common@master#egg=cloudify-common==master \ No newline at end of file diff --git a/plugin.yaml b/plugin.yaml index 6885a3f..84b9b7c 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -4,6 +4,6 @@ plugins: fabric: executor: central_deployment_agent - source: https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/2.0.1.zip + source: https://github.com/cloudify-cosmo/cloudify-fabric-plugin/archive/2.0.3.zip package_name: cloudify-fabric-plugin - package_version: '2.0.1' + package_version: '2.0.3' diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 2a9acf1..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[bdist_wheel] -universal = 1 diff --git a/setup.py b/setup.py index 7f68440..0e1bf7c 100644 --- a/setup.py +++ b/setup.py @@ -17,14 +17,14 @@ setup( name='cloudify-fabric-plugin', - version='2.0.2', + version='2.0.3', author='Cloudify', author_email='hello@cloudify.co', packages=['fabric_plugin'], license='LICENSE', description='Plugin for remotely running fabric tasks and commands', install_requires=[ - 'cloudify-common>=5.1.0.dev1', + 'cloudify-common>=4.5.5', 'fabric==2.5.0', ] ) From 6d810026f03718c776e0654bbfaf1c301b844b4a Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Sun, 21 Jun 2020 16:30:12 +0300 Subject: [PATCH 7/8] enable rhel wagon builder for testing --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9d9f27d..6ac0d7b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -176,13 +176,12 @@ jobs: steps: - checkout - generate_py27py36_wagon - # - generate_py3_wagon # TODO: When we have Python3 Manager Integration Test. rhel_wagon: executor: wagon_generator steps: - checkout -# - generate_rhel_py27py36_wagon + - generate_rhel_py27py36_wagon release: executor: py27 From f91a513e60b9eab36978d52ebd398877e0649144 Mon Sep 17 00:00:00 2001 From: AdarShaked Date: Sun, 21 Jun 2020 16:39:22 +0300 Subject: [PATCH 8/8] enable rhel wagon creator and added unitest to reqiuerments for plugin release --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6ac0d7b..94b86d9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -199,11 +199,11 @@ workflows: - wagon: filters: branches: - only: /([0-9\.]*\-build|master|dev|replace-wagon-builder-to-py2py3-builder)/ + only: /([0-9\.]*\-build|master|dev)/ - rhel_wagon: filters: branches: - only: /([0-9\.]*\-build|master|dev|replace-wagon-builder-to-py2py3-builder)/ + only: /([0-9\.]*\-build|master|dev)/ - release: filters: branches: @@ -211,5 +211,6 @@ workflows: requires: - wagon - rhel_wagon - + - unittests_py27 + - unittests_py36