Skip to content

Commit

Permalink
Merge pull request #3475 from Flamefire/refactor-find-python-cmd
Browse files Browse the repository at this point in the history 
refactor search for `python_cmd`
  • Loading branch information
@Micket
Micket authored Oct 10, 2024
2 parents 4600949 + aa0e320 commit 8382143
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 115 deletions.
63 changes: 15 additions & 48 deletions easybuild/easyblocks/generic/pythonbundle.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,12 @@
@author: Kenneth Hoste (Ghent University)
"""
import os
import sys

from easybuild.easyblocks.generic.bundle import Bundle
from easybuild.easyblocks.generic.pythonpackage import EXTS_FILTER_PYTHON_PACKAGES
from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd
from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, find_python_cmd_from_ec
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option, PYTHONPATH, EBPYTHONPREFIXES
from easybuild.tools.filetools import which
from easybuild.tools.modules import get_software_root
import easybuild.tools.environment as env

Expand Down Expand Up @@ -72,64 +70,33 @@ def __init__(self, *args, **kwargs):

self.log.info("exts_default_options: %s", self.cfg['exts_default_options'])

self.python_cmd = None
self.pylibdir = None
self.all_pylibdirs = []
self.all_pylibdirs = None

# figure out whether this bundle of Python packages is being installed for multiple Python versions
self.multi_python = 'Python' in self.cfg['multi_deps']

def prepare_step(self, *args, **kwargs):
"""Prepare for installing bundle of Python packages."""
super(Bundle, self).prepare_step(*args, **kwargs)
def prepare_python(self):
"""Python-specific preparations."""

python_root = get_software_root('Python')
if python_root is None:
if get_software_root('Python') is None:
raise EasyBuildError("Python not included as dependency!")
self.python_cmd = find_python_cmd_from_ec(self.log, self.cfg, required=True)

# when system Python is used, the first 'python' command in $PATH will not be $EBROOTPYTHON/bin/python,
# since $EBROOTPYTHON is set to just 'Python' in that case
# (see handling of allow_system_deps in EasyBlock.prepare_step)
if which('python') == os.path.join(python_root, 'bin', 'python'):
# if we're using a proper Python dependency, let det_pylibdir use 'python' like it does by default
python_cmd = None
else:
# since det_pylibdir will use 'python' by default as command to determine Python lib directory,
# we need to intervene when the system Python is used, by specifying version requirements
# to pick_python_cmd so the right 'python' command is used;
# if we're using the system Python and no Python version requirements are specified,
# use major/minor version of Python being used in this EasyBuild session (as we also do in PythonPackage)
req_py_majver = self.cfg['req_py_majver']
if req_py_majver is None:
req_py_majver = sys.version_info[0]
req_py_minver = self.cfg['req_py_minver']
if req_py_minver is None:
req_py_minver = sys.version_info[1]

# Get the max_py_majver and max_py_minver from the config
max_py_majver = self.cfg['max_py_majver']
max_py_minver = self.cfg['max_py_minver']

python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver,
max_py_majver=max_py_majver, max_py_minver=max_py_minver)

# If pick_python_cmd didn't find a (system) Python command, we should raise an error
if python_cmd:
self.log.info("Python command being used: %s", python_cmd)
else:
raise EasyBuildError(
"Failed to pick Python command that satisfies requirements in the easyconfig "
"(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)",
req_py_majver, req_py_minver, max_py_majver, max_py_minver
)

self.all_pylibdirs = get_pylibdirs(python_cmd=python_cmd)
self.all_pylibdirs = get_pylibdirs(python_cmd=self.python_cmd)
self.pylibdir = self.all_pylibdirs[0]

# if 'python' is not used, we need to take that into account in the extensions filter
# (which is also used during the sanity check)
if python_cmd:
if self.python_cmd != 'python':
orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES
self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', python_cmd), orig_exts_filter[1])
self.cfg['exts_filter'] = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1])

def prepare_step(self, *args, **kwargs):
"""Prepare for installing bundle of Python packages."""
super(Bundle, self).prepare_step(*args, **kwargs)
self.prepare_python()

def extensions_step(self, *args, **kwargs):
"""Install extensions (usually PythonPackages)"""
Expand Down
138 changes: 71 additions & 67 deletions easybuild/easyblocks/generic/pythonpackage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,31 +102,31 @@ def check_python_cmd(python_cmd):
# check whether specified Python command is available
if os.path.isabs(python_cmd):
if not os.path.isfile(python_cmd):
log.debug("Python command '%s' does not exist", python_cmd)
log.debug(f"Python command '{python_cmd}' does not exist")
return False
else:
python_cmd_path = which(python_cmd)
if python_cmd_path is None:
log.debug("Python command '%s' not available through $PATH", python_cmd)
log.debug(f"Python command '{python_cmd}' not available through $PATH")
return False

pyver = det_python_version(python_cmd)

if req_maj_ver is not None:
if req_min_ver is None:
req_majmin_ver = '%s.0' % req_maj_ver
else:
req_majmin_ver = '%s.%s' % (req_maj_ver, req_min_ver)

pyver = det_python_version(python_cmd)

# (strict) check for major version
maj_ver = pyver.split('.')[0]
if maj_ver != str(req_maj_ver):
log.debug("Major Python version does not match: %s vs %s", maj_ver, req_maj_ver)
log.debug(f"Major Python version does not match: {maj_ver} vs {req_maj_ver}")
return False

# check for minimal minor version
if LooseVersion(pyver) < LooseVersion(req_majmin_ver):
log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", pyver, req_majmin_ver)
log.debug(f"Minimal requirement for minor Python version not satisfied: {pyver} vs {req_majmin_ver}")
return False

if max_py_majver is not None:
Expand All @@ -135,45 +135,97 @@ def check_python_cmd(python_cmd):
else:
max_majmin_ver = '%s.%s' % (max_py_majver, max_py_minver)

pyver = det_python_version(python_cmd)

if LooseVersion(pyver) > LooseVersion(max_majmin_ver):
log.debug("Python version (%s) on the system is newer than the maximum supported "
"Python version specified in the easyconfig (%s)",
pyver, max_majmin_ver)
return False

# all check passed
log.debug("All check passed for Python command '%s'!", python_cmd)
log.debug(f"All check passed for Python command '{python_cmd}'!")
return True

# compose list of 'python' commands to consider
python_cmds = ['python']
if req_maj_ver:
python_cmds.append('python%s' % req_maj_ver)
python_cmds.append(f'python{req_maj_ver}')
if req_min_ver:
python_cmds.append('python%s.%s' % (req_maj_ver, req_min_ver))
python_cmds.append(f'python{req_maj_ver}.{req_min_ver}')
python_cmds.append(sys.executable)
log.debug("Considering Python commands: %s", ', '.join(python_cmds))
log.debug("Considering Python commands: " + ', '.join(python_cmds))

# try and find a 'python' command that satisfies the requirements
res = None
for python_cmd in python_cmds:
if check_python_cmd(python_cmd):
log.debug("Python command '%s' satisfies version requirements!", python_cmd)
log.debug(f"Python command '{python_cmd}' satisfies version requirements!")
if os.path.isabs(python_cmd):
res = python_cmd
else:
res = which(python_cmd)
log.debug("Absolute path to retained Python command: %s", res)
log.debug("Absolute path to retained Python command: " + res)
break
else:
log.debug("Python command '%s' does not satisfy version requirements (maj: %s, min: %s), moving on",
python_cmd, req_maj_ver, req_min_ver)
log.debug(f"Python command '{python_cmd}' does not satisfy version requirements "
f"(maj: {req_maj_ver}, min: {req_min_ver}), moving on")

return res


def find_python_cmd(log, req_py_majver, req_py_minver, max_py_majver, max_py_minver, required):
"""Return an appropriate python command to use.
When python is a dependency use the full path to that.
Else use req_py_maj/minver (defaulting to the Python being used in this EasyBuild session) to select one.
If no (matching) python command is found and raise an Error or log a warning depending on the required argument.
"""
python = None
python_root = get_software_root('Python')
# keep in mind that Python may be listed as an allowed system dependency,
# so just checking Python root is not sufficient
if python_root:
bin_python = os.path.join(python_root, 'bin', 'python')
if os.path.exists(bin_python) and os.path.samefile(which('python'), bin_python):
# if Python is listed as a (build) dependency, use 'python' command provided that way
python = bin_python
log.debug("Retaining 'python' command for Python dependency: " + python)

if python is None:
# if no Python version requirements are specified,
# use major/minor version of Python being used in this EasyBuild session
if req_py_majver is None:
req_py_majver = sys.version_info[0]
if req_py_minver is None:
req_py_minver = sys.version_info[1]
# if using system Python, go hunting for a 'python' command that satisfies the requirements
python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver,
max_py_majver=max_py_majver, max_py_minver=max_py_minver)

if python:
log.info("Python command being used: " + python)
elif required:
if all(v is None for v in (req_py_majver, req_py_minver, max_py_majver, max_py_minver)):
error_msg = "Failed to pick Python command to use"
else:
error_msg = (f"Failed to pick Python command that satisfies requirements in the easyconfig: "
f"req_py_majver = {req_py_majver}, req_py_minver = {req_py_minver}")
if max_py_majver is not None:
error_msg += f"max_py_majver = {max_py_majver}, max_py_minver = {max_py_minver}"
raise EasyBuildError(error_msg)
else:
log.warning("No Python command found!")
return python


def find_python_cmd_from_ec(log, cfg, required):
"""Find a python command using the constraints specified in the EasyConfig"""
return find_python_cmd(log,
cfg['req_py_majver'], cfg['req_py_minver'],
max_py_majver=cfg['max_py_majver'],
max_py_minver=cfg['max_py_minver'],
required=required)


def det_pylibdir(plat_specific=False, python_cmd=None):
"""Determine Python library directory."""
log = fancylogger.getLogger('det_pylibdir', fname=False)
Expand All @@ -188,8 +240,8 @@ def det_pylibdir(plat_specific=False, python_cmd=None):
if LooseVersion(det_python_version(python_cmd)) >= LooseVersion('3.12'):
# Python 3.12 removed distutils but has a core sysconfig module which is similar
pathname = 'platlib' if plat_specific else 'purelib'
vars = {'platbase': prefix, 'base': prefix}
pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars)
vars_param = {'platbase': prefix, 'base': prefix}
pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars_param)
else:
args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix)
pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args
Expand Down Expand Up @@ -503,55 +555,7 @@ def set_pylibdirs(self):
def prepare_python(self):
"""Python-specific preparations."""

# pick 'python' command to use
python = None
python_root = get_software_root('Python')
# keep in mind that Python may be listed as an allowed system dependency,
# so just checking Python root is not sufficient
if python_root:
bin_python = os.path.join(python_root, 'bin', 'python')
if os.path.exists(bin_python) and os.path.samefile(which('python'), bin_python):
# if Python is listed as a (build) dependency, use 'python' command provided that way
python = os.path.join(python_root, 'bin', 'python')
self.log.debug("Retaining 'python' command for Python dependency: %s", python)

if python is None:
# if no Python version requirements are specified,
# use major/minor version of Python being used in this EasyBuild session
req_py_majver = self.cfg['req_py_majver']
if req_py_majver is None:
req_py_majver = sys.version_info[0]
req_py_minver = self.cfg['req_py_minver']
if req_py_minver is None:
req_py_minver = sys.version_info[1]

# Get the max_py_majver and max_py_minver from the config
max_py_majver = self.cfg['max_py_majver']
max_py_minver = self.cfg['max_py_minver']

# if using system Python, go hunting for a 'python' command that satisfies the requirements
python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver,
max_py_majver=max_py_majver, max_py_minver=max_py_minver)

# Check if we have Python by now. If not, and if self.require_python, raise a sensible error
if python:
self.python_cmd = python
self.log.info("Python command being used: %s", self.python_cmd)
elif self.require_python:
if (req_py_majver is not None or req_py_minver is not None
or max_py_majver is not None or max_py_minver is not None):
raise EasyBuildError(
"Failed to pick Python command that satisfies requirements in the easyconfig "
"(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)",
req_py_majver, req_py_minver, max_py_majver, max_py_minver
)
else:
raise EasyBuildError("Failed to pick Python command to use")
else:
self.log.warning("No Python command found!")
else:
self.python_cmd = python
self.log.info("Python command being used: %s", self.python_cmd)
self.python_cmd = find_python_cmd_from_ec(self.log, self.cfg, self.require_python)

if self.python_cmd:
# set Python lib directories
Expand Down

0 comments on commit 8382143

Please sign in to comment.