diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 01d9c4c672..a92c0d410b 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -28,13 +28,11 @@ @author: Kenneth Hoste (Ghent University) """ import os -import sys from easybuild.easyblocks.generic.bundle import Bundle from easybuild.easyblocks.generic.pythonpackage import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES -from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd +from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, find_python_cmd, run_pip_check from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import which from easybuild.tools.modules import get_software_root import easybuild.tools.environment as env @@ -74,49 +72,33 @@ def __init__(self, *args, **kwargs): self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) + self.python_cmd = None self.pylibdir = None - self.all_pylibdirs = [] + self.all_pylibdirs = None # figure out whether this bundle of Python packages is being installed for multiple Python versions self.multi_python = 'Python' in self.cfg['multi_deps'] - def prepare_step(self, *args, **kwargs): - """Prepare for installing bundle of Python packages.""" - super(Bundle, self).prepare_step(*args, **kwargs) + def prepare_python(self): + """Python-specific preparations.""" - python_root = get_software_root('Python') - if python_root is None: + if get_software_root('Python') is None: raise EasyBuildError("Python not included as dependency!") + self.python_cmd = find_python_cmd(self.log, self.cfg['req_py_majver'], self.cfg['req_py_minver'], required=True) - # when system Python is used, the first 'python' command in $PATH will not be $EBROOTPYTHON/bin/python, - # since $EBROOTPYTHON is set to just 'Python' in that case - # (see handling of allow_system_deps in EasyBlock.prepare_step) - if which('python') == os.path.join(python_root, 'bin', 'python'): - # if we're using a proper Python dependency, let det_pylibdir use 'python' like it does by default - python_cmd = None - else: - # since det_pylibdir will use 'python' by default as command to determine Python lib directory, - # we need to intervene when the system Python is used, by specifying version requirements - # to pick_python_cmd so the right 'python' command is used; - # if we're using the system Python and no Python version requirements are specified, - # use major/minor version of Python being used in this EasyBuild session (as we also do in PythonPackage) - req_py_majver = self.cfg['req_py_majver'] - if req_py_majver is None: - req_py_majver = sys.version_info[0] - req_py_minver = self.cfg['req_py_minver'] - if req_py_minver is None: - req_py_minver = sys.version_info[1] - - python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver) - - self.all_pylibdirs = get_pylibdirs(python_cmd=python_cmd) + self.all_pylibdirs = get_pylibdirs(python_cmd=self.python_cmd) self.pylibdir = self.all_pylibdirs[0] # if 'python' is not used, we need to take that into account in the extensions filter # (which is also used during the sanity check) - if python_cmd: + if self.python_cmd != 'python': orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES - self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', python_cmd), orig_exts_filter[1]) + self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1]) + + def prepare_step(self, *args, **kwargs): + """Prepare for installing bundle of Python packages.""" + super(Bundle, self).prepare_step(*args, **kwargs) + self.prepare_python() def extensions_step(self, *args, **kwargs): """Install extensions (usually PythonPackages)""" @@ -158,6 +140,15 @@ def make_module_extra(self, *args, **kwargs): def sanity_check_step(self, *args, **kwargs): """Custom sanity check for bundle of Python package.""" + if self.pylibdir is None: + # Python attributes not set up yet, happens e.g. with --sanity-check-only, so do it now. + # This also ensures the exts_filter option for extensions is set correctly. + # Load module first to get the right python command. + if not self.sanity_check_module_loaded: + self.fake_mod_data = self.sanity_check_load_module(extension=kwargs.get('extension', False), + extra_modules=kwargs.get('extra_modules', None)) + self.prepare_python() + # inject directory path that uses %(pyshortver)s template into default value for sanity_check_paths # this is relevant for installations of Python bundles for multiple Python versions (via multi_deps) # (we can not pass this via custom_paths, since then the %(pyshortver)s template value will not be resolved) @@ -168,3 +159,32 @@ def sanity_check_step(self, *args, **kwargs): } super(Bundle, self).sanity_check_step(*args, **kwargs) + + def _sanity_check_step_extensions(self): + """Run the pip check for extensions if enabled""" + super(PythonBundle, self)._sanity_check_step_extensions() + + sanity_pip_check = self.cfg['sanity_pip_check'] + unversioned_packages = set(self.cfg['unversioned_packages']) + + # The options should be set in the main EC and cannot be different between extensions. + # For backwards compatibility and to avoid surprises enable the pip-check if it is enabled + # in the main EC or any extension and build the union of all unversioned_packages. + has_sanity_pip_check_mismatch = False + all_unversioned_packages = unversioned_packages.copy() + for ext in self.ext_instances: + if isinstance(ext, PythonPackage): + if ext.cfg['sanity_pip_check'] != sanity_pip_check: + has_sanity_pip_check_mismatch = True + all_unversioned_packages.update(ext.cfg['unversioned_packages']) + + if has_sanity_pip_check_mismatch: + self.log.deprecated('For bundles of PythonPackage the sanity_pip_check option ' + 'in the main EasyConfig must be used', '5.0') + sanity_pip_check = True # Either the main set it or any extension enabled it + if all_unversioned_packages != unversioned_packages: + self.log.deprecated('For bundles of PythonPackage the unversioned_packages option ' + 'in the main EasyConfig must be used', '5.0') + + if sanity_pip_check: + run_pip_check(self.log, self.python_cmd, all_unversioned_packages) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index f2ae208ebc..3793b114e2 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -161,6 +161,41 @@ def check_python_cmd(python_cmd): return res +def find_python_cmd(log, req_py_majver, req_py_minver, required): + """Return an appropriate python command to use. + + When python is a dependency prefer the full path to that. + Else use req_py_maj/minver (defaulting to the Python being used in this EasyBuild session) to select one. + If no (matching) python command is found and raise an Error or log a warning depending on the required argument. + """ + python = None + python_root = get_software_root('Python') + # keep in mind that Python may be listed as an allowed system dependency, + # so just checking Python root is not sufficient + if python_root: + bin_python = os.path.join(python_root, 'bin', 'python') + if os.path.exists(bin_python) and os.path.samefile(which('python'), bin_python): + # if Python is listed as a (build) dependency, use 'python' command provided that way + python = bin_python + log.debug("Retaining 'python' command for Python dependency: %s", python) + + if python is None: + if req_py_majver is None: + req_py_majver = sys.version_info[0] + if req_py_minver is None: + req_py_minver = sys.version_info[1] + # if using system Python, go hunting for a 'python' command that satisfies the requirements + python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver) + + if python: + log.info("Python command being used: %s", python) + elif required: + raise EasyBuildError("Failed to pick Python command to use") + else: + log.warning("No Python command found!") + return python + + def det_pylibdir(plat_specific=False, python_cmd=None): """Determine Python library directory.""" log = fancylogger.getLogger('det_pylibdir', fname=False) @@ -175,8 +210,8 @@ def det_pylibdir(plat_specific=False, python_cmd=None): if LooseVersion(det_python_version(python_cmd)) >= LooseVersion('3.12'): # Python 3.12 removed distutils but has a core sysconfig module which is similar pathname = 'platlib' if plat_specific else 'purelib' - vars = {'platbase': prefix, 'base': prefix} - pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars) + vars_param = {'platbase': prefix, 'base': prefix} + pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars_param) else: args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix) pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args @@ -323,6 +358,94 @@ def symlink_dist_site_packages(install_dir, pylibdirs): symlink(dist_pkgs, site_pkgs_path, use_abspath_source=False) +def det_installed_python_packages(log, names_only=True, python_cmd=None): + """Return list of Python packages that are installed + + When names_only is True then only the names are returned, else the full info from `pip list`. + Note that the names are reported by pip and might be different to the name that need to be used to import it + """ + # Check installed python packages but only check stdout, not stderr which might contain user facing warnings + cmd_list = [python_cmd, '-m', 'pip', 'list', '--isolated', '--disable-pip-version-check', + '--format', 'json'] + full_cmd = ' '.join(cmd_list) + log.info("Running command '%s'" % full_cmd) + proc = subprocess_popen_text(cmd_list, env=os.environ) + (stdout, stderr) = proc.communicate() + ec = proc.returncode + msg = "Command '%s' returned with %s: stdout: %s; stderr: %s" % (full_cmd, ec, stdout, stderr) + if ec: + log.info(msg) + raise EasyBuildError('Failed to determine installed python packages: %s', stderr) + + log.debug(msg) + pkgs = json.loads(stdout.strip()) + return [pkg['name'] for pkg in pkgs] if names_only else pkgs + + +def run_pip_check(log, python_cmd, unversioned_packages): + """Check installed Python packages using pip + + log - Logger + python_cmd - Python command + unversioned_packages - Python packages to exclude in the version existance check + """ + pip_check_command = "%s -m pip check" % python_cmd + pip_version = det_pip_version(python_cmd=python_cmd) + if not pip_version: + raise EasyBuildError("Failed to determine pip version!") + min_pip_version = LooseVersion('9.0.0') + if LooseVersion(pip_version) < min_pip_version: + raise EasyBuildError("pip >= %s is required for running '%s', found %s", + min_pip_version, pip_check_command, pip_version) + + pip_check_errors = [] + + pip_check_msg, ec = run_cmd(pip_check_command, log_ok=False) + if ec: + pip_check_errors.append('`%s` failed:\n%s' % (pip_check_command, pip_check_msg)) + else: + log.info('`%s` completed successfully' % pip_check_command) + + # Also check for a common issue where the package version shows up as 0.0.0 often caused + # by using setup.py as the installation method for a package which is released as a generic wheel + # named name-version-py2.py3-none-any.whl. `tox` creates those from version controlled source code + # so it will contain a version, but the raw tar.gz does not. + pkgs = det_installed_python_packages(log, names_only=False, python_cmd=python_cmd) + faulty_version = '0.0.0' + faulty_pkg_names = [pkg['name'] for pkg in pkgs if pkg['version'] == faulty_version] + + for unversioned_package in unversioned_packages: + try: + faulty_pkg_names.remove(unversioned_package) + log.debug('Excluding unversioned package %s from check', unversioned_package) + except ValueError: + try: + version = next(pkg['version'] for pkg in pkgs if pkg['name'] == unversioned_package) + except StopIteration: + msg = ('Package %s in unversioned_packages was not found in the installed packages. ' + 'Check that the name from `python -m pip list` is used which may be different ' + 'than the module name.' % unversioned_package) + else: + msg = ('Package %s in unversioned_packages has a version of %s which is valid. ' + 'Please remove it from unversioned_packages.' % (unversioned_package, version)) + pip_check_errors.append(msg) + + log.info('Found %s invalid packages out of %s packages', len(faulty_pkg_names), len(pkgs)) + if faulty_pkg_names: + msg = ( + "The following Python packages were likely not installed correctly because they show a " + "version of '%s':\n%s\n" + "This may be solved by using a *-none-any.whl file as the source instead. " + "See e.g. the SOURCE*_WHL templates.\n" + "Otherwise you could check if the package provides a version at all or if e.g. poetry is " + "required (check the source for a pyproject.toml and see PEP517 for details on that)." + ) % (faulty_version, '\n'.join(faulty_pkg_names)) + pip_check_errors.append(msg) + + if pip_check_errors: + raise EasyBuildError('\n'.join(pip_check_errors)) + + class PythonPackage(ExtensionEasyBlock): """Builds and installs a Python package, and provides a dedicated module file.""" @@ -486,38 +609,9 @@ def set_pylibdirs(self): def prepare_python(self): """Python-specific preparations.""" - # pick 'python' command to use - python = None - python_root = get_software_root('Python') - # keep in mind that Python may be listed as an allowed system dependency, - # so just checking Python root is not sufficient - if python_root: - bin_python = os.path.join(python_root, 'bin', 'python') - if os.path.exists(bin_python) and os.path.samefile(which('python'), bin_python): - # if Python is listed as a (build) dependency, use 'python' command provided that way - python = os.path.join(python_root, 'bin', 'python') - self.log.debug("Retaining 'python' command for Python dependency: %s", python) - - if python is None: - # if no Python version requirements are specified, - # use major/minor version of Python being used in this EasyBuild session - req_py_majver = self.cfg['req_py_majver'] - if req_py_majver is None: - req_py_majver = sys.version_info[0] - req_py_minver = self.cfg['req_py_minver'] - if req_py_minver is None: - req_py_minver = sys.version_info[1] - - # if using system Python, go hunting for a 'python' command that satisfies the requirements - python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver) - - if python: - self.python_cmd = python - self.log.info("Python command being used: %s", self.python_cmd) - elif self.require_python: - raise EasyBuildError("Failed to pick Python command to use") - else: - self.log.warning("No Python command found!") + self.python_cmd = find_python_cmd(self.log, + self.cfg['req_py_majver'], self.cfg['req_py_minver'], + required=self.require_python) if self.python_cmd: # set Python lib directories @@ -556,25 +650,7 @@ def get_installed_python_packages(self, names_only=True, python_cmd=None): """ if python_cmd is None: python_cmd = self.python_cmd - # Check installed python packages but only check stdout, not stderr which might contain user facing warnings - cmd_list = [python_cmd, '-m', 'pip', 'list', '--isolated', '--disable-pip-version-check', - '--format', 'json'] - full_cmd = ' '.join(cmd_list) - self.log.info("Running command '%s'" % full_cmd) - proc = subprocess_popen_text(cmd_list, env=os.environ) - (stdout, stderr) = proc.communicate() - ec = proc.returncode - msg = "Command '%s' returned with %s: stdout: %s; stderr: %s" % (full_cmd, ec, stdout, stderr) - if ec: - self.log.info(msg) - raise EasyBuildError('Failed to determine installed python packages: %s', stderr) - - self.log.debug(msg) - pkgs = json.loads(stdout.strip()) - if names_only: - return [pkg['name'] for pkg in pkgs] - else: - return pkgs + return det_installed_python_packages(self.log, names_only, python_cmd) def using_pip_install(self): """ @@ -957,10 +1033,10 @@ def sanity_check_step(self, *args, **kwargs): # load module early ourselves rather than letting parent sanity_check_step method do so, # since custom actions taken below require that environment is set up properly already # (especially when using --sanity-check-only) - if hasattr(self, 'sanity_check_module_loaded') and not self.sanity_check_module_loaded: + if not self.sanity_check_module_loaded: extension = self.is_extension or kwargs.get('extension', False) extra_modules = kwargs.get('extra_modules', None) - self.fake_mod_data = self.sanity_check_load_module(extension=extension, extra_modules=extra_modules) + self.sanity_check_load_module(extension=extension, extra_modules=extra_modules) # don't add user site directory to sys.path (equivalent to python -s) # see https://www.python.org/dev/peps/pep-0370/; @@ -1014,77 +1090,26 @@ def sanity_check_step(self, *args, **kwargs): exts_filter = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1]) kwargs.update({'exts_filter': exts_filter}) - if self.cfg.get('sanity_pip_check', False): - pip_version = det_pip_version(python_cmd=python_cmd) - - if pip_version: - pip_check_command = "%s -m pip check" % python_cmd - - if LooseVersion(pip_version) >= LooseVersion('9.0.0'): - - if not self.is_extension: - # for stand-alone Python package installations (not part of a bundle of extensions), - # the (fake or real) module file must be loaded at this point, - # otherwise the Python package being installed is not "in view", - # and we will overlook missing dependencies... - loaded_modules = [x['mod_name'] for x in self.modules_tool.list()] - if self.short_mod_name not in loaded_modules: - self.log.debug("Currently loaded modules: %s", loaded_modules) - raise EasyBuildError("%s module is not loaded, this should never happen...", - self.short_mod_name) - - pip_check_errors = [] - - pip_check_msg, ec = run_cmd(pip_check_command, log_ok=False) - if ec: - pip_check_errors.append('`%s` failed:\n%s' % (pip_check_command, pip_check_msg)) - else: - self.log.info('`%s` completed successfully' % pip_check_command) - - # Also check for a common issue where the package version shows up as 0.0.0 often caused - # by using setup.py as the installation method for a package which is released as a generic wheel - # named name-version-py2.py3-none-any.whl. `tox` creates those from version controlled source code - # so it will contain a version, but the raw tar.gz does not. - pkgs = self.get_installed_python_packages(names_only=False, python_cmd=python_cmd) - faulty_version = '0.0.0' - faulty_pkg_names = [pkg['name'] for pkg in pkgs if pkg['version'] == faulty_version] - - for unversioned_package in self.cfg.get('unversioned_packages', []): - try: - faulty_pkg_names.remove(unversioned_package) - self.log.debug('Excluding unversioned package %s from check', unversioned_package) - except ValueError: - try: - version = next(pkg['version'] for pkg in pkgs if pkg['name'] == unversioned_package) - except StopIteration: - msg = ('Package %s in unversioned_packages was not found in the installed packages. ' - 'Check that the name from `python -m pip list` is used which may be different ' - 'than the module name.' % unversioned_package) - else: - msg = ('Package %s in unversioned_packages has a version of %s which is valid. ' - 'Please remove it from unversioned_packages.' % (unversioned_package, version)) - pip_check_errors.append(msg) - - self.log.info('Found %s invalid packages out of %s packages', len(faulty_pkg_names), len(pkgs)) - if faulty_pkg_names: - msg = ( - "The following Python packages were likely not installed correctly because they show a " - "version of '%s':\n%s\n" - "This may be solved by using a *-none-any.whl file as the source instead. " - "See e.g. the SOURCE*_WHL templates.\n" - "Otherwise you could check if the package provides a version at all or if e.g. poetry is " - "required (check the source for a pyproject.toml and see PEP517 for details on that)." - ) % (faulty_version, '\n'.join(faulty_pkg_names)) - pip_check_errors.append(msg) - - if pip_check_errors: - raise EasyBuildError('\n'.join(pip_check_errors)) - else: - raise EasyBuildError("pip >= 9.0.0 is required for running '%s', found %s", - pip_check_command, - pip_version) - else: - raise EasyBuildError("Failed to determine pip version!") + sanity_pip_check = self.cfg.get('sanity_pip_check', False) + if self.is_extension: + sanity_pip_check_main = self.master.cfg.get('sanity_pip_check') + if sanity_pip_check_main is not None: + # If the main easyblock (e.g. PythonBundle) defines the variable + # we trust it does the pip check if requested and checks for mismatches + sanity_pip_check = False + + if sanity_pip_check: + if not self.is_extension: + # for stand-alone Python package installations (not part of a bundle of extensions), + # the (fake or real) module file must be loaded at this point, + # otherwise the Python package being installed is not "in view", + # and we will overlook missing dependencies... + loaded_modules = [x['mod_name'] for x in self.modules_tool.list()] + if self.short_mod_name not in loaded_modules: + self.log.debug("Currently loaded modules: %s", loaded_modules) + raise EasyBuildError("%s module is not loaded, this should never happen...", + self.short_mod_name) + run_pip_check(self.log, python_cmd, self.cfg.get('unversioned_packages', [])) # ExtensionEasyBlock handles loading modules correctly for multi_deps, so we clean up fake_mod_data # and let ExtensionEasyBlock do its job