Merge pull request #128 from boegel/py3_tests_must_pass

add support for specifying in vsc-ci.ini that tests must pass using Python 3 (and use it for vsc-install) (HPC-7603)

Jenkinsfile

@@ -10,7 +10,7 @@ node {
     }
     stage('test') {
         sh 'python2.7 -V'
-        sh 'python -m easy_install -U --user tox'
+        sh 'pip3 install --ignore-installed --user tox'
         sh 'export PATH=$HOME/.local/bin:$PATH && tox -v -c tox.ini'
     }
 }

README.md

@@ -449,3 +449,24 @@ specify this via a `vsc-ci.ini` configuration file:
 [vsc-ci]
 install_scripts_prefix_override=1
 ```
+
+Requiring that tests pass using Python 3
+----------------------------------------
+
+To require that the test suite passes when run with Python 3, you must opt-in to generating a tox configuration file
+(tox.ini) that does not ignore a missing interpreter or failing tests, using a `vsc-ci.ini` configuration file like:
+
+```ini
+[vsc-ci]
+py3_tests_must_pass=1
+```
+
+Use 'pip3' to install tox
+-------------------------
+
+On systems that have Python 3 and `pip3` installed, it is recommended to opt-in to use `pip3 install` to install tox:
+
+```ini
+[vsc-ci]
+pip3_install_tox=1
+```

lib/vsc/install/ci.py

@@ -52,6 +52,8 @@
 
 INSTALL_SCRIPTS_PREFIX_OVERRIDE = 'install_scripts_prefix_override'
 JIRA_ISSUE_ID_IN_PR_TITLE = 'jira_issue_id_in_pr_title'
+PIP3_INSTALL_TOX = 'pip3_install_tox'
+PY3_TESTS_MUST_PASS = 'py3_tests_must_pass'
 RUN_SHELLCHECK = 'run_shellcheck'
 
 logging.basicConfig(format="%(message)s", level=logging.INFO)
@@ -74,6 +76,8 @@ def gen_tox_ini():
     """
     logging.info('[%s]', TOX_INI)
 
+    vsc_ci_cfg = parse_vsc_ci_cfg()
+
     header = [
         "%s: configuration file for tox" % TOX_INI,
         "This file was automatically generated using 'python -m vsc.install.ci'",
@@ -100,9 +104,16 @@ def gen_tox_ini():
         # instruct tox not to run sdist prior to installing the package in the tox environment
         # (setup.py requires vsc-install, which is not installed yet when 'python setup.py sdist' is run)
         "skipsdist = true",
-        # ignore failures due to missing Python version
-        # python2.7 must always be available though, see Jenkinsfile
-        "skip_missing_interpreters = true",
+    ]
+
+    if not vsc_ci_cfg[PY3_TESTS_MUST_PASS]:
+        lines.extend([
+            # ignore failures due to missing Python version
+            # python2.7 must always be available though, see Jenkinsfile
+            "skip_missing_interpreters = true",
+        ])
+
+    lines.extend([
         '',
         '[testenv]',
         "commands_pre =",
@@ -120,11 +131,15 @@ def gen_tox_ini():
         # $USER is not defined in tox environment, so pass it
         # see https://tox.readthedocs.io/en/latest/example/basic.html#passing-down-environment-variables
         'passenv = USER',
-        '',
-        # allow failing tests in Python 3, for now...
-        '[testenv:%s]' % py3_env,
-        "ignore_outcome = true"
-    ]
+    ])
+
+    if not vsc_ci_cfg[PY3_TESTS_MUST_PASS]:
+        lines.extend([
+            '',
+            # allow failing tests in Python 3, for now...
+            '[testenv:%s]' % py3_env,
+            "ignore_outcome = true"
+        ])
 
     return '\n'.join(lines) + '\n'
 
@@ -134,6 +149,8 @@ def parse_vsc_ci_cfg():
     vsc_ci_cfg = {
         INSTALL_SCRIPTS_PREFIX_OVERRIDE: False,
         JIRA_ISSUE_ID_IN_PR_TITLE: False,
+        PIP3_INSTALL_TOX: False,
+        PY3_TESTS_MUST_PASS: False,
         RUN_SHELLCHECK: False,
     }
 
@@ -174,11 +191,16 @@ def indent(line, level=1):
         # since we've configured tox to ignore failures due to missing Python interpreters
         # (see skip_missing_interpreters in gen_tox_ini)
         'python2.7 -V',
-        'python -m easy_install -U --user tox',
-        # make sure 'tox' command installed with --user is available via $PATH
-        'export PATH=$HOME/.local/bin:$PATH && tox -v -c %s' % TOX_INI,
     ]
 
+    if vsc_ci_cfg[PIP3_INSTALL_TOX]:
+        test_cmds.append('pip3 install --ignore-installed --user tox')
+    else:
+        test_cmds.append('python -m easy_install -U --user tox')
+
+    # make sure 'tox' command installed with --user is available via $PATH/$PYTHONPATH
+    test_cmds.append('export PATH=$HOME/.local/bin:$PATH && tox -v -c %s' % TOX_INI)
+
     header = [
         "%s: scripted Jenkins pipefile" % JENKINSFILE,
         "This file was automatically generated using 'python -m vsc.install.ci'",

lib/vsc/install/shared_setup.py

@@ -159,7 +159,7 @@ def _log(self, level, msg, args):
 
 RELOAD_VSC_MODS = False
 
-VERSION = '0.14.13'
+VERSION = '0.15.0'
 
 log.info('This is (based on) vsc.install.shared_setup %s' % VERSION)
 log.info('(using setuptools version %s located at %s)' % (setuptools.__version__, setuptools.__file__))

test/ci.py

@@ -96,7 +96,9 @@
     python -m easy_install -U vsc-install
 commands = python setup.py test
 passenv = USER
+"""
 
+EXPECTED_TOX_INI_PY36_IGNORE = """
 [testenv:py36]
 ignore_outcome = true
 """
@@ -122,30 +124,26 @@ def write_vsc_ci_ini(self, txt):
     def test_parse_vsc_ci_cfg(self):
         """Test parse_vsc_ci_cfg function."""
 
+        keys = [
+            'install_scripts_prefix_override',
+            'jira_issue_id_in_pr_title',
+            'pip3_install_tox',
+            'py3_tests_must_pass',
+            'run_shellcheck',
+        ]
+
         # (basically) empty vsc-ci.ini
         self.write_vsc_ci_ini('')
-        expected = {
-            'install_scripts_prefix_override': False,
-            'jira_issue_id_in_pr_title': False,
-            'run_shellcheck': False,
-        }
+        expected = dict((key, False) for key in keys)
         self.assertEqual(parse_vsc_ci_cfg(), expected)
 
         # vsc-ci.ini with unknown keys is trouble
         self.write_vsc_ci_ini("unknown_key=1")
         error_msg = "Unknown key in vsc-ci.ini: unknown_key"
         self.assertErrorRegex(ValueError, error_msg, parse_vsc_ci_cfg)
 
-        self.write_vsc_ci_ini('\n'.join([
-            'install_scripts_prefix_override=1',
-            'jira_issue_id_in_pr_title=1',
-            'run_shellcheck=true',
-        ]))
-        expected = {
-            'install_scripts_prefix_override': True,
-            'jira_issue_id_in_pr_title': True,
-            'run_shellcheck': True,
-        }
+        self.write_vsc_ci_ini('\n'.join('%s=1' % key for key in keys))
+        expected = dict((key, True) for key in keys)
         self.assertEqual(parse_vsc_ci_cfg(), expected)
 
     def test_gen_jenkinsfile(self):
@@ -169,14 +167,22 @@ def test_gen_jenkinsfile_shellcheck(self):
 
     def test_tox_ini(self):
         """Test generating of tox.ini."""
-        self.assertEqual(gen_tox_ini(), EXPECTED_TOX_INI)
+        self.assertEqual(gen_tox_ini(), EXPECTED_TOX_INI + EXPECTED_TOX_INI_PY36_IGNORE)
+
+    def test_tox_ini_py3_tests(self):
+        """Test generation of tox.ini when Python 3 tests are expected to pass."""
+
+        self.write_vsc_ci_ini('py3_tests_must_pass=1')
+
+        expected = EXPECTED_TOX_INI.replace('skip_missing_interpreters = true\n', '')
+        self.assertEqual(gen_tox_ini(), expected)
 
     def test_tox_ini_install_script(self):
         """Test generating of tox.ini when install_scripts_prefix_override is set."""
 
         self.write_vsc_ci_ini('install_scripts_prefix_override=1')
 
-        expected = EXPECTED_TOX_INI
+        expected = EXPECTED_TOX_INI + EXPECTED_TOX_INI_PY36_IGNORE
         pip_regex = re.compile('pip install')
         expected = pip_regex.sub('pip install --install-option="--install-scripts={envdir}/bin"', expected)
         easy_install_regex = re.compile('easy_install -U')

tox.ini

@@ -5,14 +5,10 @@
 [tox]
 envlist = py27,py36
 skipsdist = true
-skip_missing_interpreters = true
 
 [testenv]
 commands_pre =
     pip install 'setuptools<42.0'
     python -m easy_install -U vsc-install
 commands = python setup.py test
 passenv = USER
-
-[testenv:py36]
-ignore_outcome = true

vsc-ci.ini

@@ -0,0 +1,3 @@
+[vsc-ci]
+pip3_install_tox=1
+py3_tests_must_pass=1