diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 76729c4105..d85579e356 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -28,14 +28,12 @@ @author: Kenneth Hoste (Ghent University) """ import os -import sys from easybuild.easyblocks.generic.bundle import Bundle from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_PYTHON_PACKAGES -from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd +from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, find_python_cmd_from_ec from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES -from easybuild.tools.filetools import which from easybuild.tools.modules import get_software_root import easybuild.tools.environment as env @@ -72,64 +70,33 @@ def __init__(self, *args, **kwargs): self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) + self.python_cmd = None self.pylibdir = None - self.all_pylibdirs = [] + self.all_pylibdirs = None # figure out whether this bundle of Python packages is being installed for multiple Python versions self.multi_python = 'Python' in self.cfg['multi_deps'] - def prepare_step(self, *args, **kwargs): - """Prepare for installing bundle of Python packages.""" - super(Bundle, self).prepare_step(*args, **kwargs) + def prepare_python(self): + """Python-specific preparations.""" - python_root = get_software_root('Python') - if python_root is None: + if get_software_root('Python') is None: raise EasyBuildError("Python not included as dependency!") + self.python_cmd = find_python_cmd_from_ec(self.log, self.cfg, required=True) - # when system Python is used, the first 'python' command in $PATH will not be $EBROOTPYTHON/bin/python, - # since $EBROOTPYTHON is set to just 'Python' in that case - # (see handling of allow_system_deps in EasyBlock.prepare_step) - if which('python') == os.path.join(python_root, 'bin', 'python'): - # if we're using a proper Python dependency, let det_pylibdir use 'python' like it does by default - python_cmd = None - else: - # since det_pylibdir will use 'python' by default as command to determine Python lib directory, - # we need to intervene when the system Python is used, by specifying version requirements - # to pick_python_cmd so the right 'python' command is used; - # if we're using the system Python and no Python version requirements are specified, - # use major/minor version of Python being used in this EasyBuild session (as we also do in PythonPackage) - req_py_majver = self.cfg['req_py_majver'] - if req_py_majver is None: - req_py_majver = sys.version_info[0] - req_py_minver = self.cfg['req_py_minver'] - if req_py_minver is None: - req_py_minver = sys.version_info[1] - - # Get the max_py_majver and max_py_minver from the config - max_py_majver = self.cfg['max_py_majver'] - max_py_minver = self.cfg['max_py_minver'] - - python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, - max_py_majver=max_py_majver, max_py_minver=max_py_minver) - - # If pick_python_cmd didn't find a (system) Python command, we should raise an error - if python_cmd: - self.log.info("Python command being used: %s", python_cmd) - else: - raise EasyBuildError( - "Failed to pick Python command that satisfies requirements in the easyconfig " - "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", - req_py_majver, req_py_minver, max_py_majver, max_py_minver - ) - - self.all_pylibdirs = get_pylibdirs(python_cmd=python_cmd) + self.all_pylibdirs = get_pylibdirs(python_cmd=self.python_cmd) self.pylibdir = self.all_pylibdirs[0] # if 'python' is not used, we need to take that into account in the extensions filter # (which is also used during the sanity check) - if python_cmd: + if self.python_cmd != 'python': orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES - self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', python_cmd), orig_exts_filter[1]) + self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1]) + + def prepare_step(self, *args, **kwargs): + """Prepare for installing bundle of Python packages.""" + super(Bundle, self).prepare_step(*args, **kwargs) + self.prepare_python() def extensions_step(self, *args, **kwargs): """Install extensions (usually PythonPackages)""" diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 1b9a8ea517..eda88a21ec 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -102,31 +102,31 @@ def check_python_cmd(python_cmd): # check whether specified Python command is available if os.path.isabs(python_cmd): if not os.path.isfile(python_cmd): - log.debug("Python command '%s' does not exist", python_cmd) + log.debug(f"Python command '{python_cmd}' does not exist") return False else: python_cmd_path = which(python_cmd) if python_cmd_path is None: - log.debug("Python command '%s' not available through $PATH", python_cmd) + log.debug(f"Python command '{python_cmd}' not available through $PATH") return False + pyver = det_python_version(python_cmd) + if req_maj_ver is not None: if req_min_ver is None: req_majmin_ver = '%s.0' % req_maj_ver else: req_majmin_ver = '%s.%s' % (req_maj_ver, req_min_ver) - pyver = det_python_version(python_cmd) - # (strict) check for major version maj_ver = pyver.split('.')[0] if maj_ver != str(req_maj_ver): - log.debug("Major Python version does not match: %s vs %s", maj_ver, req_maj_ver) + log.debug(f"Major Python version does not match: {maj_ver} vs {req_maj_ver}") return False # check for minimal minor version if LooseVersion(pyver) < LooseVersion(req_majmin_ver): - log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", pyver, req_majmin_ver) + log.debug(f"Minimal requirement for minor Python version not satisfied: {pyver} vs {req_majmin_ver}") return False if max_py_majver is not None: @@ -135,8 +135,6 @@ def check_python_cmd(python_cmd): else: max_majmin_ver = '%s.%s' % (max_py_majver, max_py_minver) - pyver = det_python_version(python_cmd) - if LooseVersion(pyver) > LooseVersion(max_majmin_ver): log.debug("Python version (%s) on the system is newer than the maximum supported " "Python version specified in the easyconfig (%s)", @@ -144,36 +142,90 @@ def check_python_cmd(python_cmd): return False # all check passed - log.debug("All check passed for Python command '%s'!", python_cmd) + log.debug(f"All check passed for Python command '{python_cmd}'!") return True # compose list of 'python' commands to consider python_cmds = ['python'] if req_maj_ver: - python_cmds.append('python%s' % req_maj_ver) + python_cmds.append(f'python{req_maj_ver}') if req_min_ver: - python_cmds.append('python%s.%s' % (req_maj_ver, req_min_ver)) + python_cmds.append(f'python{req_maj_ver}.{req_min_ver}') python_cmds.append(sys.executable) - log.debug("Considering Python commands: %s", ', '.join(python_cmds)) + log.debug("Considering Python commands: " + ', '.join(python_cmds)) # try and find a 'python' command that satisfies the requirements res = None for python_cmd in python_cmds: if check_python_cmd(python_cmd): - log.debug("Python command '%s' satisfies version requirements!", python_cmd) + log.debug(f"Python command '{python_cmd}' satisfies version requirements!") if os.path.isabs(python_cmd): res = python_cmd else: res = which(python_cmd) - log.debug("Absolute path to retained Python command: %s", res) + log.debug("Absolute path to retained Python command: " + res) break else: - log.debug("Python command '%s' does not satisfy version requirements (maj: %s, min: %s), moving on", - python_cmd, req_maj_ver, req_min_ver) + log.debug(f"Python command '{python_cmd}' does not satisfy version requirements " + f"(maj: {req_maj_ver}, min: {req_min_ver}), moving on") return res +def find_python_cmd(log, req_py_majver, req_py_minver, max_py_majver, max_py_minver, required): + """Return an appropriate python command to use. + + When python is a dependency use the full path to that. + Else use req_py_maj/minver (defaulting to the Python being used in this EasyBuild session) to select one. + If no (matching) python command is found and raise an Error or log a warning depending on the required argument. + """ + python = None + python_root = get_software_root('Python') + # keep in mind that Python may be listed as an allowed system dependency, + # so just checking Python root is not sufficient + if python_root: + bin_python = os.path.join(python_root, 'bin', 'python') + if os.path.exists(bin_python) and os.path.samefile(which('python'), bin_python): + # if Python is listed as a (build) dependency, use 'python' command provided that way + python = bin_python + log.debug("Retaining 'python' command for Python dependency: " + python) + + if python is None: + # if no Python version requirements are specified, + # use major/minor version of Python being used in this EasyBuild session + if req_py_majver is None: + req_py_majver = sys.version_info[0] + if req_py_minver is None: + req_py_minver = sys.version_info[1] + # if using system Python, go hunting for a 'python' command that satisfies the requirements + python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, + max_py_majver=max_py_majver, max_py_minver=max_py_minver) + + if python: + log.info("Python command being used: " + python) + elif required: + if all(v is None for v in (req_py_majver, req_py_minver, max_py_majver, max_py_minver)): + error_msg = "Failed to pick Python command to use" + else: + error_msg = (f"Failed to pick Python command that satisfies requirements in the easyconfig: " + f"req_py_majver = {req_py_majver}, req_py_minver = {req_py_minver}") + if max_py_majver is not None: + error_msg += f"max_py_majver = {max_py_majver}, max_py_minver = {max_py_minver}" + raise EasyBuildError(error_msg) + else: + log.warning("No Python command found!") + return python + + +def find_python_cmd_from_ec(log, cfg, required): + """Find a python command using the constraints specified in the EasyConfig""" + return find_python_cmd(log, + cfg['req_py_majver'], cfg['req_py_minver'], + max_py_majver=cfg['max_py_majver'], + max_py_minver=cfg['max_py_minver'], + required=required) + + def det_pylibdir(plat_specific=False, python_cmd=None): """Determine Python library directory.""" log = fancylogger.getLogger('det_pylibdir', fname=False) @@ -188,8 +240,8 @@ def det_pylibdir(plat_specific=False, python_cmd=None): if LooseVersion(det_python_version(python_cmd)) >= LooseVersion('3.12'): # Python 3.12 removed distutils but has a core sysconfig module which is similar pathname = 'platlib' if plat_specific else 'purelib' - vars = {'platbase': prefix, 'base': prefix} - pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars) + vars_param = {'platbase': prefix, 'base': prefix} + pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars_param) else: args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix) pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args @@ -503,55 +555,7 @@ def set_pylibdirs(self): def prepare_python(self): """Python-specific preparations.""" - # pick 'python' command to use - python = None - python_root = get_software_root('Python') - # keep in mind that Python may be listed as an allowed system dependency, - # so just checking Python root is not sufficient - if python_root: - bin_python = os.path.join(python_root, 'bin', 'python') - if os.path.exists(bin_python) and os.path.samefile(which('python'), bin_python): - # if Python is listed as a (build) dependency, use 'python' command provided that way - python = os.path.join(python_root, 'bin', 'python') - self.log.debug("Retaining 'python' command for Python dependency: %s", python) - - if python is None: - # if no Python version requirements are specified, - # use major/minor version of Python being used in this EasyBuild session - req_py_majver = self.cfg['req_py_majver'] - if req_py_majver is None: - req_py_majver = sys.version_info[0] - req_py_minver = self.cfg['req_py_minver'] - if req_py_minver is None: - req_py_minver = sys.version_info[1] - - # Get the max_py_majver and max_py_minver from the config - max_py_majver = self.cfg['max_py_majver'] - max_py_minver = self.cfg['max_py_minver'] - - # if using system Python, go hunting for a 'python' command that satisfies the requirements - python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver, - max_py_majver=max_py_majver, max_py_minver=max_py_minver) - - # Check if we have Python by now. If not, and if self.require_python, raise a sensible error - if python: - self.python_cmd = python - self.log.info("Python command being used: %s", self.python_cmd) - elif self.require_python: - if (req_py_majver is not None or req_py_minver is not None - or max_py_majver is not None or max_py_minver is not None): - raise EasyBuildError( - "Failed to pick Python command that satisfies requirements in the easyconfig " - "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)", - req_py_majver, req_py_minver, max_py_majver, max_py_minver - ) - else: - raise EasyBuildError("Failed to pick Python command to use") - else: - self.log.warning("No Python command found!") - else: - self.python_cmd = python - self.log.info("Python command being used: %s", self.python_cmd) + self.python_cmd = find_python_cmd_from_ec(self.log, self.cfg, self.require_python) if self.python_cmd: # set Python lib directories