From 6eb24a9dac0d8b64dd5a364a72a39d6661ad465b Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Fri, 20 Sep 2024 11:57:02 -0500 Subject: [PATCH 1/2] Split selector logic into conda_build.selectors --- conda_build/jinja_context.py | 35 +++- conda_build/metadata.py | 311 +++++++++-------------------------- conda_build/selectors.py | 230 ++++++++++++++++++++++++++ conda_build/utils.py | 2 + conda_build/variants.py | 15 +- tests/test_metadata.py | 242 +++------------------------ tests/test_selectors.py | 230 ++++++++++++++++++++++++++ 7 files changed, 600 insertions(+), 465 deletions(-) create mode 100644 conda_build/selectors.py create mode 100644 tests/test_selectors.py diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index 307a13ecc9..db2eaab0f2 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -19,6 +19,7 @@ from frozendict import deepfreeze from . import _load_setup_py_data +from .deprecations import deprecated from .environ import get_dict as get_environ from .exceptions import CondaBuildException from .render import get_env_dependencies @@ -39,7 +40,11 @@ import tomli as tomllib if TYPE_CHECKING: - from typing import IO, Any + from typing import IO, Any, Callable, Sequence + + from jinja2 import BaseLoader, Environment + + from .config import Config log = get_logger(__name__) @@ -113,6 +118,11 @@ def _return_value(self, value=None): return value +@deprecated( + "24.11", + "25.1", + addendum="Use `conda_build.jinja_context.FilteredChoiceLoader` instead.", +) class FilteredLoader(jinja2.BaseLoader): """ A pass-through for the given loader, except that the loaded source is @@ -142,6 +152,29 @@ def get_source(self, environment, template): ) +class FilteredChoiceLoader(jinja2.ChoiceLoader): + def __init__(self, loaders: Sequence[BaseLoader], config: Config) -> None: + super().__init__(loaders) + self.config = config + + def get_source( + self, environment: Environment, template: str + ) -> tuple[str, str | None, Callable[[], bool] | None]: + # we have circular imports here. Do a local import + from .metadata import get_selectors, select_lines + + contents, filename, uptodate = super().get_source(environment, template) + return ( + select_lines( + contents, + get_selectors(self.config), + variants_in_place=bool(self.config.variant), + ), + filename, + uptodate, + ) + + def load_setup_py_data( m, setup_file="setup.py", diff --git a/conda_build/metadata.py b/conda_build/metadata.py index b744a8b3c9..d5b86f947e 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -9,7 +9,6 @@ import re import sys import time -import warnings from collections import OrderedDict from functools import lru_cache from os.path import isdir, isfile, join @@ -24,6 +23,7 @@ from . import utils from .config import Config, get_or_merge_config +from .deprecations import deprecated from .exceptions import ( CondaBuildException, CondaBuildUserError, @@ -32,10 +32,13 @@ UnableToParse, UnableToParseMissingJinja2, ) -from .features import feature_list from .license_family import ensure_valid_license_family +from .selectors import RE_SELECTOR, _split_line_selector, parse_NameError +from .selectors import eval_selector as _eval_selector +from .selectors import get_selectors as _get_selectors +from .selectors import select_lines as _select_lines +from .utils import ARCH_MAP as _ARCH_MAP from .utils import ( - DEFAULT_SUBDIRS, ensure_list, expand_globs, find_recipe, @@ -48,7 +51,6 @@ find_used_variables_in_batch_script, find_used_variables_in_shell_script, find_used_variables_in_text, - get_default_variant, get_vars, list_of_dicts_to_dict_of_lists, ) @@ -100,8 +102,13 @@ def remove_constructor(cls, tag): StringifyNumbersLoader.remove_constructor("tag:yaml.org,2002:float") StringifyNumbersLoader.remove_constructor("tag:yaml.org,2002:int") -# arches that don't follow exact names in the subdir need to be mapped here -ARCH_MAP = {"32": "x86", "64": "x86_64"} +deprecated.constant( + "24.11", + "25.1", + "ARCH_MAP", + _ARCH_MAP, + addendum="Use `conda_build.utils.ARCH_MAP` instead.", +) NOARCH_TYPES = ("python", "generic", True) @@ -131,227 +138,58 @@ def remove_constructor(cls, tag): # used to avoid recomputing/rescanning recipe contents for used variables used_vars_cache = {} - -def get_selectors(config: Config) -> dict[str, bool]: - """Aggregates selectors for use in recipe templating. - - Derives selectors from the config and variants to be injected - into the Jinja environment prior to templating. - - Args: - config (Config): The config object - - Returns: - dict[str, bool]: Dictionary of on/off selectors for Jinja - """ - # Remember to update the docs of any of this changes - plat = config.host_subdir - d = dict( - linux32=bool(plat == "linux-32"), - linux64=bool(plat == "linux-64"), - arm=plat.startswith("linux-arm"), - unix=plat.startswith(("linux-", "osx-", "emscripten-")), - win32=bool(plat == "win-32"), - win64=bool(plat == "win-64"), - os=os, - environ=os.environ, - nomkl=bool(int(os.environ.get("FEATURE_NOMKL", False))), - ) - - # Add the current platform to the list of subdirs to enable conda-build - # to bootstrap new platforms without a new conda release. - subdirs = list(DEFAULT_SUBDIRS) + [plat] - - # filter out noarch and other weird subdirs - subdirs = [subdir for subdir in subdirs if "-" in subdir] - - subdir_oses = {subdir.split("-")[0] for subdir in subdirs} - subdir_archs = {subdir.split("-")[1] for subdir in subdirs} - - for subdir_os in subdir_oses: - d[subdir_os] = plat.startswith(f"{subdir_os}-") - - for arch in subdir_archs: - arch_full = ARCH_MAP.get(arch, arch) - d[arch_full] = plat.endswith(f"-{arch}") - if arch == "32": - d["x86"] = plat.endswith(("-32", "-64")) - - defaults = get_default_variant(config) - py = config.variant.get("python", defaults["python"]) - # there are times when python comes in as a tuple - if not hasattr(py, "split"): - py = py[0] - # go from "3.6 *_cython" -> "36" - # or from "3.6.9" -> "36" - py_major, py_minor, *_ = py.split(" ")[0].split(".") - py = int(f"{py_major}{py_minor}") - - d["build_platform"] = config.build_subdir - - d.update( - dict( - py=py, - py3k=bool(py_major == "3"), - py2k=bool(py_major == "2"), - py26=bool(py == 26), - py27=bool(py == 27), - py33=bool(py == 33), - py34=bool(py == 34), - py35=bool(py == 35), - py36=bool(py == 36), - ) - ) - - np = config.variant.get("numpy") - if not np: - np = defaults["numpy"] - if config.verbose: - utils.get_logger(__name__).warning( - "No numpy version specified in conda_build_config.yaml. " - "Falling back to default numpy value of {}".format(defaults["numpy"]) - ) - d["np"] = int("".join(np.split(".")[:2])) - - pl = config.variant.get("perl", defaults["perl"]) - d["pl"] = pl - - lua = config.variant.get("lua", defaults["lua"]) - d["lua"] = lua - d["luajit"] = bool(lua[0] == "2") - - for feature, value in feature_list: - d[feature] = value - d.update(os.environ) - - # here we try to do some type conversion for more intuitive usage. Otherwise, - # values like 35 are strings by default, making relational operations confusing. - # We also convert "True" and things like that to booleans. - for k, v in config.variant.items(): - if k not in d: - try: - d[k] = int(v) - except (TypeError, ValueError): - if isinstance(v, str) and v.lower() in ("false", "true"): - v = v.lower() == "true" - d[k] = v - return d - - -def ns_cfg(config: Config) -> dict[str, bool]: - warnings.warn( - "`conda_build.metadata.ns_cfg` is pending deprecation and will be removed in a " - "future release. Please use `conda_build.metadata.get_selectors` instead.", - PendingDeprecationWarning, - ) - return get_selectors(config) - - -# Selectors must be either: -# - at end of the line -# - embedded (anywhere) within a comment -# -# Notes: -# - [([^\[\]]+)\] means "find a pair of brackets containing any -# NON-bracket chars, and capture the contents" -# - (?(2)[^\(\)]*)$ means "allow trailing characters iff group 2 (#.*) was found." -# Skip markdown link syntax. -sel_pat = re.compile(r"(.+?)\s*(#.*)?\[([^\[\]]+)\](?(2)[^\(\)]*)$") - - -# this function extracts the variable name from a NameError exception, it has the form of: -# "NameError: name 'var' is not defined", where var is the variable that is not defined. This gets -# returned -def parseNameNotFound(error): - m = re.search("'(.+?)'", str(error)) - if len(m.groups()) == 1: - return m.group(1) - else: - return "" - - -# We evaluate the selector and return True (keep this line) or False (drop this line) -# If we encounter a NameError (unknown variable in selector), then we replace it by False and -# re-run the evaluation -def eval_selector(selector_string, namespace, variants_in_place): - try: - # TODO: is there a way to do this without eval? Eval allows arbitrary - # code execution. - return eval(selector_string, namespace, {}) - except NameError as e: - missing_var = parseNameNotFound(e) - if variants_in_place: - log = utils.get_logger(__name__) - log.debug( - "Treating unknown selector '" + missing_var + "' as if it was False." - ) - next_string = selector_string.replace(missing_var, "False") - return eval_selector(next_string, namespace, variants_in_place) - - -@lru_cache(maxsize=None) -def _split_line_selector(text: str) -> tuple[tuple[str | None, str], ...]: - lines: list[tuple[str | None, str]] = [] - for line in text.splitlines(): - line = line.rstrip() - - # skip comment lines, include a blank line as a placeholder - if line.lstrip().startswith("#"): - lines.append((None, "")) - continue - - # include blank lines - if not line: - lines.append((None, "")) - continue - - # user may have quoted entire line to make YAML happy - trailing_quote = "" - if line and line[-1] in ("'", '"'): - trailing_quote = line[-1] - - # Checking for "[" and "]" before regex matching every line is a bit faster. - if ( - ("[" in line and "]" in line) - and (match := sel_pat.match(line)) - and (selector := match.group(3)) - ): - # found a selector - lines.append((selector, (match.group(1) + trailing_quote).rstrip())) - else: - # no selector found - lines.append((None, line)) - return tuple(lines) - - -def select_lines(text: str, namespace: dict[str, Any], variants_in_place: bool) -> str: - lines = [] - selector_cache: dict[str, bool] = {} - for i, (selector, line) in enumerate(_split_line_selector(text)): - if not selector: - # no selector? include line as is - lines.append(line) - else: - # include lines with a selector that evaluates to True - try: - if selector_cache[selector]: - lines.append(line) - except KeyError: - # KeyError: cache miss - try: - value = bool(eval_selector(selector, namespace, variants_in_place)) - selector_cache[selector] = value - if value: - lines.append(line) - except Exception as e: - raise CondaBuildUserError( - f"Invalid selector in meta.yaml line {i + 1}:\n" - f"offending selector:\n" - f" [{selector}]\n" - f"exception:\n" - f" {e.__class__.__name__}: {e}\n" - ) - return "\n".join(lines) + "\n" +deprecated.constant( + "24.11", + "25.1", + "get_selectors", + _get_selectors, + addendum="Use `conda_build.selectors.get_selectors` instead.", +) +deprecated.constant( + "24.11", + "25.1", + "ns_cfg", + _get_selectors, + addendum="Use `conda_build.selectors.get_selectors` instead.", +) +deprecated.constant( + "24.11", + "25.1", + "sel_pat", + RE_SELECTOR, + addendum="Use `conda_build.selectors.RE_SELECTOR` instead.", +) +del RE_SELECTOR +deprecated.constant( + "24.11", + "25.1", + "parseNameNotFound", + parse_NameError, + addendum="Use `conda_build.selectors.parse_NameError` instead.", +) +del parse_NameError +deprecated.constant( + "24.11", + "25.1", + "eval_selector", + _eval_selector, + addendum="Use `conda_build.selectors.eval_selector` instead.", +) +deprecated.constant( + "24.11", + "25.1", + "_split_line_selector", + _split_line_selector, + addendum="Use `conda_build.selectors._split_line_selector` instead.", +) +del _split_line_selector +deprecated.constant( + "24.11", + "25.1", + "select_lines", + _select_lines, + addendum="Use `conda_build.selectors.select_lines` instead.", +) def yamlize(data): @@ -519,9 +357,9 @@ def ensure_matching_hashes(output_metadata): def parse(data, config, path=None): - data = select_lines( + data = _select_lines( data, - get_selectors(config), + _get_selectors(config), variants_in_place=bool(config.variant), ) res = yamlize(data) @@ -1778,7 +1616,7 @@ def info_index(self): platform=self.config.host_platform if (self.config.host_platform != "noarch" and arch != "noarch") else None, - arch=ARCH_MAP.get(arch, arch), + arch=_ARCH_MAP.get(arch, arch), subdir=self.config.target_subdir, depends=sorted(" ".join(ms.spec.split()) for ms in self.ms_depends()), timestamp=int(time.time() * 1000), @@ -1930,13 +1768,14 @@ def _get_contents( return fd.read() from .jinja_context import ( - FilteredLoader, + FilteredChoiceLoader, UndefinedNeverFail, context_processor, ) path, filename = os.path.split(self.meta_path) - loaders = [ # search relative to '/Lib/site-packages/conda_build/templates' + loaders = [ + # search relative to '/Lib/site-packages/conda_build/templates' jinja2.PackageLoader("conda_build"), # search relative to RECIPE_DIR jinja2.FileSystemLoader(path), @@ -1959,12 +1798,12 @@ def _get_contents( UndefinedNeverFail.all_undefined_names = [] undefined_type = UndefinedNeverFail - loader = FilteredLoader(jinja2.ChoiceLoader(loaders), config=self.config) + loader = FilteredChoiceLoader(loaders, config=self.config) env = jinja2.Environment(loader=loader, undefined=undefined_type) from .environ import get_dict - env.globals.update(get_selectors(self.config)) + env.globals.update(_get_selectors(self.config)) env.globals.update(get_dict(m=self, skip_build_id=skip_build_id)) env.globals.update({"CONDA_BUILD_STATE": "RENDER"}) env.globals.update( @@ -2151,9 +1990,9 @@ def get_recipe_text( recipe_text = output_yaml(self) recipe_text = _filter_recipe_text(recipe_text, extract_pattern) if apply_selectors: - recipe_text = select_lines( + recipe_text = _select_lines( recipe_text, - get_selectors(self.config), + _get_selectors(self.config), variants_in_place=bool(self.config.variant), ) return recipe_text.rstrip() diff --git a/conda_build/selectors.py b/conda_build/selectors.py new file mode 100644 index 0000000000..2ef31ea6fc --- /dev/null +++ b/conda_build/selectors.py @@ -0,0 +1,230 @@ +# Copyright (C) 2014 Anaconda, Inc +# SPDX-License-Identifier: BSD-3-Clause +from __future__ import annotations + +import os +import re +from functools import lru_cache +from typing import TYPE_CHECKING + +from .exceptions import CondaBuildUserError +from .features import feature_list +from .utils import ARCH_MAP, DEFAULT_SUBDIRS, get_logger +from .variants import get_default_variant + +if TYPE_CHECKING: + from typing import Any + + from .config import Config + + +def get_selectors(config: Config) -> dict[str, bool]: + """Aggregates selectors for use in recipe templating. + + Derives selectors from the config and variants to be injected + into the Jinja environment prior to templating. + + Args: + config (Config): The config object + + Returns: + dict[str, bool]: Dictionary of on/off selectors for Jinja + """ + # Remember to update the docs of any of this changes + plat = config.host_subdir + d = dict( + linux32=bool(plat == "linux-32"), + linux64=bool(plat == "linux-64"), + arm=plat.startswith("linux-arm"), + unix=plat.startswith(("linux-", "osx-", "emscripten-")), + win32=bool(plat == "win-32"), + win64=bool(plat == "win-64"), + os=os, + environ=os.environ, + nomkl=bool(int(os.environ.get("FEATURE_NOMKL", False))), + ) + + # Add the current platform to the list of subdirs to enable conda-build + # to bootstrap new platforms without a new conda release. + subdirs = list(DEFAULT_SUBDIRS) + [plat] + + # filter out noarch and other weird subdirs + subdirs = [subdir for subdir in subdirs if "-" in subdir] + + subdir_oses = {subdir.split("-")[0] for subdir in subdirs} + subdir_archs = {subdir.split("-")[1] for subdir in subdirs} + + for subdir_os in subdir_oses: + d[subdir_os] = plat.startswith(f"{subdir_os}-") + + for arch in subdir_archs: + arch_full = ARCH_MAP.get(arch, arch) + d[arch_full] = plat.endswith(f"-{arch}") + if arch == "32": + d["x86"] = plat.endswith(("-32", "-64")) + + defaults = get_default_variant(config) + py = config.variant.get("python", defaults["python"]) + # there are times when python comes in as a tuple + if not hasattr(py, "split"): + py = py[0] + # go from "3.6 *_cython" -> "36" + # or from "3.6.9" -> "36" + py_major, py_minor, *_ = py.split(" ")[0].split(".") + py = int(f"{py_major}{py_minor}") + + d["build_platform"] = config.build_subdir + + d.update( + dict( + py=py, + py3k=bool(py_major == "3"), + py2k=bool(py_major == "2"), + py26=bool(py == 26), + py27=bool(py == 27), + py33=bool(py == 33), + py34=bool(py == 34), + py35=bool(py == 35), + py36=bool(py == 36), + ) + ) + + np = config.variant.get("numpy") + if not np: + np = defaults["numpy"] + if config.verbose: + get_logger(__name__).warning( + "No numpy version specified in conda_build_config.yaml. " + "Falling back to default numpy value of {}".format(defaults["numpy"]) + ) + d["np"] = int("".join(np.split(".")[:2])) + + pl = config.variant.get("perl", defaults["perl"]) + d["pl"] = pl + + lua = config.variant.get("lua", defaults["lua"]) + d["lua"] = lua + d["luajit"] = bool(lua[0] == "2") + + for feature, value in feature_list: + d[feature] = value + d.update(os.environ) + + # here we try to do some type conversion for more intuitive usage. Otherwise, + # values like 35 are strings by default, making relational operations confusing. + # We also convert "True" and things like that to booleans. + for k, v in config.variant.items(): + if k not in d: + try: + d[k] = int(v) + except (TypeError, ValueError): + if isinstance(v, str) and v.lower() in ("false", "true"): + v = v.lower() == "true" + d[k] = v + return d + + +# this function extracts the variable name from a NameError exception, it has the form of: +# "NameError: name 'var' is not defined", where var is the variable that is not defined. This gets +# returned +def parse_NameError(error: NameError) -> str: + if match := re.search("'(.+?)'", str(error)): + return match.group(1) + return "" + + +# We evaluate the selector and return True (keep this line) or False (drop this line) +# If we encounter a NameError (unknown variable in selector), then we replace it by False and +# re-run the evaluation +def eval_selector( + selector_string: str, namespace: dict[str, Any], variants_in_place: bool +) -> bool: + try: + # TODO: is there a way to do this without eval? Eval allows arbitrary + # code execution. + return eval(selector_string, namespace, {}) + except NameError as e: + missing_var = parse_NameError(e) + if variants_in_place: + get_logger(__name__).debug( + f"Treating unknown selector {missing_var!r} as if it was False." + ) + next_string = selector_string.replace(missing_var, "False") + return eval_selector(next_string, namespace, variants_in_place) + + +# Selectors must be either: +# - at end of the line +# - embedded (anywhere) within a comment +# +# Notes: +# - [([^\[\]]+)\] means "find a pair of brackets containing any +# NON-bracket chars, and capture the contents" +# - (?(2)[^\(\)]*)$ means "allow trailing characters iff group 2 (#.*) was found." +# Skip markdown link syntax. +RE_SELECTOR = re.compile(r"(.+?)\s*(#.*)?\[([^\[\]]+)\](?(2)[^\(\)]*)$") + + +@lru_cache(maxsize=None) +def _split_line_selector(text: str) -> tuple[tuple[str | None, str], ...]: + lines: list[tuple[str | None, str]] = [] + for line in text.splitlines(): + line = line.rstrip() + + # skip comment lines, include a blank line as a placeholder + if line.lstrip().startswith("#"): + lines.append((None, "")) + continue + + # include blank lines + if not line: + lines.append((None, "")) + continue + + # user may have quoted entire line to make YAML happy + trailing_quote = "" + if line and line[-1] in ("'", '"'): + trailing_quote = line[-1] + + # Checking for "[" and "]" before regex matching every line is a bit faster. + if ( + ("[" in line and "]" in line) + and (match := RE_SELECTOR.match(line)) + and (selector := match.group(3)) + ): + # found a selector + lines.append((selector, (match.group(1) + trailing_quote).rstrip())) + else: + # no selector found + lines.append((None, line)) + return tuple(lines) + + +def select_lines(text: str, namespace: dict[str, Any], variants_in_place: bool) -> str: + lines = [] + selector_cache: dict[str, bool] = {} + for i, (selector, line) in enumerate(_split_line_selector(text)): + if not selector: + # no selector? include line as is + lines.append(line) + else: + # include lines with a selector that evaluates to True + try: + if selector_cache[selector]: + lines.append(line) + except KeyError: + # KeyError: cache miss + try: + value = bool(eval_selector(selector, namespace, variants_in_place)) + selector_cache[selector] = value + if value: + lines.append(line) + except Exception as e: + raise CondaBuildUserError( + f"Invalid selector in meta.yaml line {i + 1}:\n" + f"offending selector:\n" + f" [{selector}]\n" + f"exception:\n" + f" {e.__class__.__name__}: {e}\n" + ) + return "\n".join(lines) + "\n" diff --git a/conda_build/utils.py b/conda_build/utils.py index 4b5fdcc8d2..b698b8feaf 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -88,6 +88,8 @@ mmap_PROT_WRITE = 0 if on_win else mmap.PROT_WRITE DEFAULT_SUBDIRS = set(KNOWN_SUBDIRS) +# arches that don't follow exact names in the subdir need to be mapped here +ARCH_MAP = {"32": "x86", "64": "x86_64"} RUN_EXPORTS_TYPES = { "weak", diff --git a/conda_build/variants.py b/conda_build/variants.py index 8a23f27405..bf868eeb72 100644 --- a/conda_build/variants.py +++ b/conda_build/variants.py @@ -24,6 +24,8 @@ if TYPE_CHECKING: from typing import Any, Iterable + from .config import Config + DEFAULT_VARIANTS = { "python": f"{sys.version_info.major}.{sys.version_info.minor}", "numpy": { @@ -136,13 +138,14 @@ def get_default_variant(config): return base -def parse_config_file(path, config): - from .metadata import get_selectors, select_lines +def parse_config_file( + path: str | os.PathLike[str] | Path, config: Config +) -> dict[str, Any]: + from .selectors import get_selectors, select_lines - with open(path) as f: - contents = f.read() - contents = select_lines(contents, get_selectors(config), variants_in_place=False) - content = yaml.load(contents, Loader=yaml.loader.BaseLoader) or {} + text = Path(path).read_text() + text = select_lines(text, get_selectors(config), variants_in_place=False) + content = yaml.load(text, Loader=yaml.loader.BaseLoader) or {} trim_empty_keys(content) return content diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 99b26dd424..c06c5b9cc4 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -4,18 +4,12 @@ import os import subprocess -import sys from contextlib import nullcontext -from itertools import product from typing import TYPE_CHECKING import pytest -from conda import __version__ as conda_version -from conda.base.context import context -from packaging.version import Version -from conda_build import api -from conda_build.config import Config +from conda_build import api, metadata from conda_build.exceptions import CondaBuildUserError from conda_build.metadata import ( FIELDS, @@ -23,21 +17,16 @@ MetaData, _hash_dependencies, check_bad_chrs, - get_selectors, sanitize, - select_lines, yamlize, ) from conda_build.utils import DEFAULT_SUBDIRS -from conda_build.variants import DEFAULT_VARIANTS from .utils import metadata_dir, metadata_path, thisdir if TYPE_CHECKING: from pathlib import Path - from pytest import MonkeyPatch - def test_uses_vcs_in_metadata(testing_workdir, testing_metadata): testing_metadata._meta_path = os.path.join(testing_workdir, "meta.yaml") @@ -60,111 +49,6 @@ def test_uses_vcs_in_metadata(testing_workdir, testing_metadata): assert not testing_metadata.uses_vcs_in_build -def test_select_lines(): - lines = "\n".join( - ( - "", # preserve leading newline - "test", - "test [abc] no", - "test [abc] # no", - " ' test ' ", - ' " test " ', - "", # preserve newline - "# comment line", # preserve comment line (but not the comment) - "test [abc]", - " 'quoted # [abc] '", - ' "quoted # [abc] yes "', - "test # stuff [abc] yes", - "test {{ JINJA_VAR[:2] }}", - "test {{ JINJA_VAR[:2] }} # stuff [abc] yes", - "test {{ JINJA_VAR[:2] }} # stuff yes [abc]", - "test {{ JINJA_VAR[:2] }} # [abc] stuff yes", - '{{ environ["test"] }} # [abc]', - "", # preserve trailing newline - ) - ) - - assert select_lines(lines, {"abc": True}, variants_in_place=True) == "\n".join( - ( - "", # preserve leading newline - "test", - "test [abc] no", - "test [abc] # no", - " ' test '", - ' " test "', - "", # preserve newline - "", # preserve comment line (but not the comment) - "test", - " 'quoted'", - ' "quoted"', - "test", - "test {{ JINJA_VAR[:2] }}", - "test {{ JINJA_VAR[:2] }}", - "test {{ JINJA_VAR[:2] }}", - "test {{ JINJA_VAR[:2] }}", - '{{ environ["test"] }}', - "", # preserve trailing newline - ) - ) - assert select_lines(lines, {"abc": False}, variants_in_place=True) == "\n".join( - ( - "", # preserve leading newline - "test", - "test [abc] no", - "test [abc] # no", - " ' test '", - ' " test "', - "", # preserve newline - "", # preserve comment line (but not the comment) - "test {{ JINJA_VAR[:2] }}", - "", # preserve trailing newline - ) - ) - - -@pytest.mark.benchmark -def test_select_lines_battery(): - test_foo = "test [foo]" - test_bar = "test [bar]" - test_baz = "test [baz]" - test_foo_and_bar = "test [foo and bar]" - test_foo_and_baz = "test [foo and baz]" - test_foo_or_bar = "test [foo or bar]" - test_foo_or_baz = "test [foo or baz]" - - lines = "\n".join( - ( - test_foo, - test_bar, - test_baz, - test_foo_and_bar, - test_foo_and_baz, - test_foo_or_bar, - test_foo_or_baz, - ) - * 10 - ) - - for _ in range(10): - for foo, bar, baz in product((True, False), repeat=3): - namespace = {"foo": foo, "bar": bar, "baz": baz} - selection = ( - ["test"] - * ( - foo - + bar - + baz - + (foo and bar) - + (foo and baz) - + (foo or bar) - + (foo or baz) - ) - * 10 - ) - selection = "\n".join(selection) + "\n" # trailing newline - assert select_lines(lines, namespace, variants_in_place=True) == selection - - def test_disallow_leading_period_in_version(testing_metadata): testing_metadata.meta["package"]["version"] = ".ste.ve" testing_metadata.final = True @@ -443,103 +327,6 @@ def test_yamlize_versions(): assert yml == ["1.2.3", "1.2.3.4"] -OS_ARCH: tuple[str, ...] = ( - "aarch64", - "arm", - "arm64", - "armv6l", - "armv7l", - "linux", - "linux32", - "linux64", - "osx", - "ppc64", - "ppc64le", - "s390x", - "unix", - "win", - "win32", - "win64", - "x86", - "x86_64", - "z", - "zos", -) - -if Version(conda_version) >= Version("23.3"): - OS_ARCH = (*OS_ARCH, "riscv64") - -if Version(conda_version) >= Version("23.7"): - OS_ARCH = (*OS_ARCH, "freebsd") - -if Version(conda_version) >= Version("23.9"): - OS_ARCH = (*OS_ARCH, "emscripten", "wasi", "wasm32") - - -@pytest.mark.parametrize( - ( - "subdir", # defined in conda.base.constants.KNOWN_SUBDIRS - "expected", # OS_ARCH keys expected to be True - ), - [ - ("emscripten-wasm32", {"unix", "emscripten", "wasm32"}), - ("wasi-wasm32", {"wasi", "wasm32"}), - ("freebsd-64", {"freebsd", "x86", "x86_64"}), - ("linux-32", {"unix", "linux", "linux32", "x86"}), - ("linux-64", {"unix", "linux", "linux64", "x86", "x86_64"}), - ("linux-aarch64", {"unix", "linux", "aarch64"}), - ("linux-armv6l", {"unix", "linux", "arm", "armv6l"}), - ("linux-armv7l", {"unix", "linux", "arm", "armv7l"}), - ("linux-ppc64", {"unix", "linux", "ppc64"}), - ("linux-ppc64le", {"unix", "linux", "ppc64le"}), - ("linux-riscv64", {"unix", "linux", "riscv64"}), - ("linux-s390x", {"unix", "linux", "s390x"}), - ("osx-64", {"unix", "osx", "x86", "x86_64"}), - ("osx-arm64", {"unix", "osx", "arm64"}), - ("win-32", {"win", "win32", "x86"}), - ("win-64", {"win", "win64", "x86", "x86_64"}), - ("win-arm64", {"win", "arm64"}), - ("zos-z", {"zos", "z"}), - ], -) -@pytest.mark.parametrize("nomkl", [0, 1]) -def test_get_selectors( - monkeypatch: MonkeyPatch, - subdir: str, - expected: set[str], - nomkl: int, -): - monkeypatch.setenv("FEATURE_NOMKL", str(nomkl)) - - config = Config(host_subdir=subdir) - assert get_selectors(config) == { - # defaults - "build_platform": context.subdir, - "lua": DEFAULT_VARIANTS["lua"], - "luajit": DEFAULT_VARIANTS["lua"] == 2, - "np": int(float(DEFAULT_VARIANTS["numpy"]) * 100), - "os": os, - "pl": DEFAULT_VARIANTS["perl"], - "py": int(f"{sys.version_info.major}{sys.version_info.minor}"), - "py26": sys.version_info[:2] == (2, 6), - "py27": sys.version_info[:2] == (2, 7), - "py2k": sys.version_info.major == 2, - "py33": sys.version_info[:2] == (3, 3), - "py34": sys.version_info[:2] == (3, 4), - "py35": sys.version_info[:2] == (3, 5), - "py36": sys.version_info[:2] == (3, 6), - "py3k": sys.version_info.major == 3, - "nomkl": bool(nomkl), - # default OS/arch values - **{key: False for key in OS_ARCH}, - # environment variables - "environ": os.environ, - **os.environ, - # override with True values - **{key: True for key in expected}, - } - - def test_fromstring(): MetaData.fromstring((metadata_path / "multiple_sources" / "meta.yaml").read_text()) @@ -559,14 +346,6 @@ def test_get_section(testing_metadata: MetaData): assert isinstance(section, dict) -def test_select_lines_invalid(): - with pytest.raises( - CondaBuildUserError, - match=r"Invalid selector in meta\.yaml", - ): - select_lines("text # [{bad]", {}, variants_in_place=True) - - @pytest.mark.parametrize( "keys,expected", [ @@ -647,3 +426,22 @@ def test_parse_until_resolved_skip_avoids_undefined_jinja( pytest.fail( "Undefined variable caused error, even though this build is skipped" ) + + +@pytest.mark.parametrize( + "function,raises", + [ + ("ARCH_MAP", TypeError), + ("get_selectors", TypeError), + ("ns_cfg", TypeError), + ("sel_pat", TypeError), + ("parseNameNotFound", TypeError), + ("eval_selector", TypeError), + ("_split_line_selector", TypeError), + ("select_lines", TypeError), + ], +) +def test_deprecations(function: str, raises: type[Exception] | None) -> None: + raises_context = pytest.raises(raises) if raises else nullcontext() + with pytest.deprecated_call(), raises_context: + getattr(metadata, function)() diff --git a/tests/test_selectors.py b/tests/test_selectors.py new file mode 100644 index 0000000000..cd3310246e --- /dev/null +++ b/tests/test_selectors.py @@ -0,0 +1,230 @@ +# Copyright (C) 2014 Anaconda, Inc +# SPDX-License-Identifier: BSD-3-Clause +from __future__ import annotations + +import os +import sys +from itertools import product +from typing import TYPE_CHECKING + +import pytest +from conda import __version__ as conda_version +from conda.base.context import context +from packaging.version import Version + +from conda_build.config import Config +from conda_build.exceptions import CondaBuildUserError +from conda_build.selectors import get_selectors, select_lines +from conda_build.variants import DEFAULT_VARIANTS + +if TYPE_CHECKING: + from pytest import MonkeyPatch + + +def test_select_lines() -> None: + lines = "\n".join( + ( + "", # preserve leading newline + "test", + "test [abc] no", + "test [abc] # no", + " ' test ' ", + ' " test " ', + "", # preserve newline + "# comment line", # preserve comment line (but not the comment) + "test [abc]", + " 'quoted # [abc] '", + ' "quoted # [abc] yes "', + "test # stuff [abc] yes", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }} # stuff [abc] yes", + "test {{ JINJA_VAR[:2] }} # stuff yes [abc]", + "test {{ JINJA_VAR[:2] }} # [abc] stuff yes", + '{{ environ["test"] }} # [abc]', + "", # preserve trailing newline + ) + ) + + assert select_lines(lines, {"abc": True}, variants_in_place=True) == "\n".join( + ( + "", # preserve leading newline + "test", + "test [abc] no", + "test [abc] # no", + " ' test '", + ' " test "', + "", # preserve newline + "", # preserve comment line (but not the comment) + "test", + " 'quoted'", + ' "quoted"', + "test", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }}", + "test {{ JINJA_VAR[:2] }}", + '{{ environ["test"] }}', + "", # preserve trailing newline + ) + ) + assert select_lines(lines, {"abc": False}, variants_in_place=True) == "\n".join( + ( + "", # preserve leading newline + "test", + "test [abc] no", + "test [abc] # no", + " ' test '", + ' " test "', + "", # preserve newline + "", # preserve comment line (but not the comment) + "test {{ JINJA_VAR[:2] }}", + "", # preserve trailing newline + ) + ) + + +@pytest.mark.benchmark +def test_select_lines_battery() -> None: + test_foo = "test [foo]" + test_bar = "test [bar]" + test_baz = "test [baz]" + test_foo_and_bar = "test [foo and bar]" + test_foo_and_baz = "test [foo and baz]" + test_foo_or_bar = "test [foo or bar]" + test_foo_or_baz = "test [foo or baz]" + + lines = "\n".join( + ( + test_foo, + test_bar, + test_baz, + test_foo_and_bar, + test_foo_and_baz, + test_foo_or_bar, + test_foo_or_baz, + ) + * 10 + ) + + for _ in range(10): + for foo, bar, baz in product((True, False), repeat=3): + namespace = {"foo": foo, "bar": bar, "baz": baz} + selection = ( + "\n".join( + ["test"] + * ( + foo + + bar + + baz + + (foo and bar) + + (foo and baz) + + (foo or bar) + + (foo or baz) + ) + * 10 + ) + + "\n" + ) # trailing newline + assert select_lines(lines, namespace, variants_in_place=True) == selection + + +OS_ARCH: tuple[str, ...] = ( + "aarch64", + "arm", + "arm64", + "armv6l", + "armv7l", + "linux", + "linux32", + "linux64", + "osx", + "ppc64", + "ppc64le", + "s390x", + "unix", + "win", + "win32", + "win64", + "x86", + "x86_64", + "z", + "zos", +) + +if Version(conda_version) >= Version("23.3"): + OS_ARCH = (*OS_ARCH, "riscv64") + +if Version(conda_version) >= Version("23.7"): + OS_ARCH = (*OS_ARCH, "freebsd") + +if Version(conda_version) >= Version("23.9"): + OS_ARCH = (*OS_ARCH, "emscripten", "wasi", "wasm32") + + +@pytest.mark.parametrize( + ( + "subdir", # defined in conda.base.constants.KNOWN_SUBDIRS + "expected", # OS_ARCH keys expected to be True + ), + [ + ("emscripten-wasm32", {"unix", "emscripten", "wasm32"}), + ("wasi-wasm32", {"wasi", "wasm32"}), + ("freebsd-64", {"freebsd", "x86", "x86_64"}), + ("linux-32", {"unix", "linux", "linux32", "x86"}), + ("linux-64", {"unix", "linux", "linux64", "x86", "x86_64"}), + ("linux-aarch64", {"unix", "linux", "aarch64"}), + ("linux-armv6l", {"unix", "linux", "arm", "armv6l"}), + ("linux-armv7l", {"unix", "linux", "arm", "armv7l"}), + ("linux-ppc64", {"unix", "linux", "ppc64"}), + ("linux-ppc64le", {"unix", "linux", "ppc64le"}), + ("linux-riscv64", {"unix", "linux", "riscv64"}), + ("linux-s390x", {"unix", "linux", "s390x"}), + ("osx-64", {"unix", "osx", "x86", "x86_64"}), + ("osx-arm64", {"unix", "osx", "arm64"}), + ("win-32", {"win", "win32", "x86"}), + ("win-64", {"win", "win64", "x86", "x86_64"}), + ("win-arm64", {"win", "arm64"}), + ("zos-z", {"zos", "z"}), + ], +) +@pytest.mark.parametrize("nomkl", [0, 1]) +def test_get_selectors( + monkeypatch: MonkeyPatch, + subdir: str, + expected: set[str], + nomkl: int, +) -> None: + monkeypatch.setenv("FEATURE_NOMKL", str(nomkl)) + + config = Config(host_subdir=subdir) + assert get_selectors(config) == { + # defaults + "build_platform": context.subdir, + "lua": DEFAULT_VARIANTS["lua"], + "luajit": DEFAULT_VARIANTS["lua"] == 2, + "np": int(float(DEFAULT_VARIANTS["numpy"]) * 100), + "os": os, + "pl": DEFAULT_VARIANTS["perl"], + "py": int(f"{sys.version_info.major}{sys.version_info.minor}"), + "py26": sys.version_info[:2] == (2, 6), + "py27": sys.version_info[:2] == (2, 7), + "py2k": sys.version_info.major == 2, + "py33": sys.version_info[:2] == (3, 3), + "py34": sys.version_info[:2] == (3, 4), + "py35": sys.version_info[:2] == (3, 5), + "py36": sys.version_info[:2] == (3, 6), + "py3k": sys.version_info.major == 3, + "nomkl": bool(nomkl), + # default OS/arch values + **{key: False for key in OS_ARCH}, + # environment variables + "environ": os.environ, + **os.environ, + # override with True values + **{key: True for key in expected}, + } + + +def test_select_lines_invalid() -> None: + with pytest.raises(CondaBuildUserError, match=r"Invalid selector in meta\.yaml"): + select_lines("text # [{bad]", {}, variants_in_place=True) From 6a5ab5f978c8bfcd5c5fbeca1dcc1dbd896d1e3e Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Fri, 20 Sep 2024 12:04:42 -0500 Subject: [PATCH 2/2] Add news --- news/5500-conda_build.selectors | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 news/5500-conda_build.selectors diff --git a/news/5500-conda_build.selectors b/news/5500-conda_build.selectors new file mode 100644 index 0000000000..1bd1cc517f --- /dev/null +++ b/news/5500-conda_build.selectors @@ -0,0 +1,26 @@ +### Enhancements + +* Split selector logic from `conda_build.metadata` into a dedicated `conda_build.selectors` module. (#5500) + +### Bug fixes + +* + +### Deprecations + +* Deprecate `conda_build.metadata.ARCH_MAP`. Use `conda_build.utils.ARCH_MAP` instead. (#5500) +* Deprecate `conda_build.metadata.get_selectors`. Use `conda_build.selectors.get_selectors` instead. (#5500) +* Deprecate `conda_build.metadata.ns_cfg`. Use `conda_build.selectors.get_selectors` instead. (#5500) +* Deprecate `conda_build.metadata.sel_pat`. Use `conda_build.selectors.RE_SELECTOR` instead. (#5500) +* Deprecate `conda_build.metadata.parseNameNotFound`. Use `conda_build.selectors.parse_NameError` instead. (#5500) +* Deprecate `conda_build.metadata.eval_selector`. Use `conda_build.selectors.eval_selector` instead. (#5500) +* Deprecate `conda_build.metadata._split_line_selector`. Use `conda_build.selectors._split_line_selector` instead. (#5500) +* Deprecate `conda_build.metadata.select_lines`. Use `conda_build.selectors.select_lines` instead. (#5500) + +### Docs + +* + +### Other + +*