From 18dca18981d68da34bb6b9ee87dd1379ca61fee2 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 16 Dec 2024 00:04:30 +0000 Subject: [PATCH 1/2] Use ``resource_path()`` to access datatypes_conf.xml.sample as a package resource This required other changes included here as well: 1) Extend `parse_xml()` to accept a `Traversable` (i.e. the output of `resource_path()`. Open the file before passing it to `etree.parse()`, since lxml's implementation probably doesn't support a `Traversable`. This also simplifies the exception handling, since the workaround for the missing `errno` in the `OSError` exception is not needed any more. ------ 2) Fix Galaxy startup when the ``data_manager_config_file`` config option is set to an empty value When does this happen? ---------------------- planemo (which we use in the converters tests) set this option to an empty value when not testing data managers: https://github.com/galaxyproject/planemo/blob/master/planemo/galaxy/config.py#L452 This empty config value was resolved to the `path_resolves_to` directory for this option. With the change in 1) above, this results in the following traceback at Galaxy startup: ``` Traceback (most recent call last): File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/galaxy/webapps/galaxy/buildapp.py", line 68, in app_pair app = galaxy.app.UniverseApplication(global_conf=global_conf, is_webapp=True, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/galaxy/app.py", line 777, in __init__ self.data_managers = self._register_singleton(DataManagers) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/galaxy/di.py", line 27, in _register_singleton instance = self[dep_type] ~~~~^^^^^^^^^^ File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/.venv/lib/python3.12/site-packages/lagom/container.py", line 381, in __getitem__ return self.resolve(dep) ^^^^^^^^^^^^^^^^^ File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/.venv/lib/python3.12/site-packages/lagom/container.py", line 265, in resolve return self._reflection_build_with_err_handling(dep_type, suppress_error) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/.venv/lib/python3.12/site-packages/lagom/container.py", line 392, in _reflection_build_with_err_handling return self._reflection_build(dep_type) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/.venv/lib/python3.12/site-packages/lagom/container.py", line 408, in _reflection_build return dep_type(**sub_deps) # type: ignore ^^^^^^^^^^^^^^^^^^^^ File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/galaxy/tools/data_manager/manager.py", line 44, in __init__ self.load_from_xml(filename) File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/galaxy/tools/data_manager/manager.py", line 58, in load_from_xml tree = util.parse_xml(xml_filename) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/users/ga002/soranzon/software/nsoranzo_galaxy/lib/galaxy/util/__init__.py", line 360, in parse_xml with source.open("rb") as f: ^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.12/pathlib.py", line 1013, in open return io.open(self, mode, buffering, encoding, errors, newline) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ IsADirectoryError: [Errno 21] Is a directory: '/tmp/tmps2qq5htq' ``` Why is this a problem now? -------------------------- Before this commit, `util.parse_xml()` was raising a `XMLSyntaxError` exception if passed a directory, but now it raises an `IsADirectoryError`, which is a subclass of `OSError` with errno 21. As a consequence, when `load_from_xml()` in lib/galaxy/tools/data_manager/manager.py calls `util.parse_xml()` passing a directory, the resulting exception is now re-raised in the `except OSError as e:` branch instead of just being logged, which is actually good since this is a misconfiguration. Solutions -------- - Do not resolve an empty/None path to the `path_resolves_to` directory. - Do not check empty/None paths against root - Adjst unit tests accordingly - Raise an OSError if the `job_config_file` config option is set to null (previously this would be resolved to a directory, also raising an OSError when trying to open that as a file). N.B.: This will also need to be fixed in Planemo by not setting the `data_manager_config_file` option when not needed. --- lib/galaxy/config/__init__.py | 8 +++++--- lib/galaxy/jobs/__init__.py | 2 ++ lib/galaxy/tool_util/linters/datatypes.py | 7 +++++-- lib/galaxy/util/__init__.py | 25 +++++++++++++++-------- test/unit/config/test_path_graph.py | 6 +++--- test/unit/config/test_path_resolves_to.py | 8 ++++---- 6 files changed, 36 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 1dc273bc7240..ef284bae1158 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -512,10 +512,10 @@ def resolve(key): if not parent: # base case: nothing else needs resolving return path parent_path = resolve(parent) # recursively resolve parent path - if path is not None: + if path: path = os.path.join(parent_path, path) # resolve path else: - path = parent_path # or use parent path + log.warning("Trying to resolve path for the '%s' option but it's empty/None", key) setattr(self, key, path) # update property _cache[key] = path # cache it! @@ -539,7 +539,7 @@ def resolve(key): if self.is_set(key) and self.paths_to_check_against_root and key in self.paths_to_check_against_root: self._check_against_root(key) - def _check_against_root(self, key): + def _check_against_root(self, key: str): def get_path(current_path, initial_path): # if path does not exist and was set as relative: if not self._path_exists(current_path) and not os.path.isabs(initial_path): @@ -558,6 +558,8 @@ def get_path(current_path, initial_path): return current_path current_value = getattr(self, key) # resolved path or list of resolved paths + if not current_value: + return if isinstance(current_value, list): initial_paths = listify(self._raw_config[key], do_strip=True) # initial unresolved paths updated_paths = [] diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index c26aa11dbc0a..ddfb5f01102f 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -387,6 +387,8 @@ def __init__(self, app: MinimalManagerApp): f" release of Galaxy. Please convert to YAML at {self.app.config.job_config_file} or" f" explicitly set `job_config_file` to {job_config_file} to remove this message" ) + if not job_config_file: + raise OSError() if ".xml" in job_config_file: tree = load(job_config_file) job_config_dict = self.__parse_job_conf_xml(tree) diff --git a/lib/galaxy/tool_util/linters/datatypes.py b/lib/galaxy/tool_util/linters/datatypes.py index 534868ed01ec..f05ae5c7ab23 100644 --- a/lib/galaxy/tool_util/linters/datatypes.py +++ b/lib/galaxy/tool_util/linters/datatypes.py @@ -2,6 +2,7 @@ from typing import ( Set, TYPE_CHECKING, + Union, ) # from galaxy import config @@ -10,15 +11,17 @@ listify, parse_xml, ) +from galaxy.util.resources import resource_path if TYPE_CHECKING: from galaxy.tool_util.lint import LintContext from galaxy.tool_util.parser import ToolSource + from galaxy.util.resources import Traversable -DATATYPES_CONF = os.path.join(os.path.dirname(__file__), "datatypes_conf.xml.sample") +DATATYPES_CONF = resource_path(__package__, "datatypes_conf.xml.sample") -def _parse_datatypes(datatype_conf_path: str) -> Set[str]: +def _parse_datatypes(datatype_conf_path: Union[str, "Traversable"]) -> Set[str]: datatypes = set() tree = parse_xml(datatype_conf_path) root = tree.getroot() diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 7445a06bb3aa..7a0690412c71 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -45,6 +45,7 @@ Optional, overload, Tuple, + TYPE_CHECKING, TypeVar, Union, ) @@ -75,6 +76,7 @@ LXML_AVAILABLE = True try: from lxml import etree + from lxml.etree import DocumentInvalid # lxml.etree.Element is a function that returns a new instance of the # lxml.etree._Element class. This class doesn't have a proper __init__() @@ -123,6 +125,10 @@ def XML(text: Union[str, bytes]) -> Element: XML, ) + class DocumentInvalid(Exception): # type: ignore[no-redef] + pass + + from . import requests from .custom_logging import get_logger from .inflection import Inflector @@ -142,6 +148,9 @@ def shlex_join(split_command): return " ".join(map(shlex.quote, split_command)) +if TYPE_CHECKING: + from galaxy.util.resources import Traversable + inflector = Inflector() log = get_logger(__name__) @@ -333,7 +342,10 @@ def unique_id(KEY_SIZE=128): def parse_xml( - fname: StrPath, strip_whitespace=True, remove_comments=True, schemafname: Union[StrPath, None] = None + fname: Union[StrPath, "Traversable"], + strip_whitespace: bool = True, + remove_comments: bool = True, + schemafname: Union[StrPath, None] = None, ) -> ElementTree: """Returns a parsed xml tree""" parser = None @@ -348,8 +360,10 @@ def parse_xml( schema_root = etree.XML(schema_file.read()) schema = etree.XMLSchema(schema_root) + source = Path(fname) if isinstance(fname, (str, os.PathLike)) else fname try: - tree = cast(ElementTree, etree.parse(str(fname), parser=parser)) + with source.open("rb") as f: + tree = cast(ElementTree, etree.parse(f, parser=parser)) root = tree.getroot() if strip_whitespace: for elem in root.iter("*"): @@ -359,15 +373,10 @@ def parse_xml( elem.tail = elem.tail.strip() if schema: schema.assertValid(tree) - except OSError as e: - if e.errno is None and not os.path.exists(fname): # type: ignore[unreachable] - # lxml doesn't set errno - e.errno = errno.ENOENT # type: ignore[unreachable] - raise except etree.ParseError: log.exception("Error parsing file %s", fname) raise - except etree.DocumentInvalid: + except DocumentInvalid: log.exception("Validation of file %s failed", fname) raise return tree diff --git a/test/unit/config/test_path_graph.py b/test/unit/config/test_path_graph.py index 02f33787081f..77afe7333c55 100644 --- a/test/unit/config/test_path_graph.py +++ b/test/unit/config/test_path_graph.py @@ -142,7 +142,7 @@ def test_resolves_to_invalid_type(monkeypatch): def test_resolves_with_empty_component(monkeypatch): - # A path can be None (root path is never None; may be asigned elsewhere) + # A path can be None (root path is never None; may be assigned elsewhere) mock_schema = { "path0": { "type": "str", @@ -155,7 +155,7 @@ def test_resolves_with_empty_component(monkeypatch): "path2": { "type": "str", "default": "value2", - "path_resolves_to": "path1", + "path_resolves_to": "path0", }, } monkeypatch.setattr(AppSchema, "_read_schema", lambda a, b: get_schema(mock_schema)) @@ -163,5 +163,5 @@ def test_resolves_with_empty_component(monkeypatch): config = BaseAppConfiguration() assert config.path0 == "value0" - assert config.path1 == "value0" + assert config.path1 is None assert config.path2 == "value0/value2" diff --git a/test/unit/config/test_path_resolves_to.py b/test/unit/config/test_path_resolves_to.py index 97953b3d0787..d7e7dd0918d6 100644 --- a/test/unit/config/test_path_resolves_to.py +++ b/test/unit/config/test_path_resolves_to.py @@ -132,21 +132,21 @@ def test_kwargs_relative_path_old_prefix_for_other_option(mock_init): def test_kwargs_relative_path_old_prefix_empty_after_strip(mock_init): # Expect: use value from kwargs, strip old prefix, then resolve - new_path1 = "old-config" + new_path1 = "old-config/foo" config = BaseAppConfiguration(path1=new_path1) - assert config.path1 == "my-config/" # stripped of old prefix, then resolved + assert config.path1 == "my-config/foo" # stripped of old prefix, then resolved assert config.path2 == "my-data/my-data-files" # stripped of old prefix, then resolved assert config.path3 == "my-other-files" # no change def test_kwargs_set_to_null(mock_init): - # Expected: allow overriding with null, then resolve + # Expected: allow overriding with null # This is not a common scenario, but it does happen: one example is # `job_config` set to `None` when testing config = BaseAppConfiguration(path1=None) - assert config.path1 == "my-config" # resolved + assert config.path1 is None assert config.path2 == "my-data/my-data-files" # resolved assert config.path3 == "my-other-files" # no change From 78234720c4e25dc07dd056570fb216bb69feb0ee Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 13 Dec 2024 12:41:28 +0000 Subject: [PATCH 2/2] Replace deprecated ``__package__`` with ``__name__` `__package__` is deprecated since Python 3.13, see https://docs.python.org/3/reference/datamodel.html#module.__package__ `importlib.resources.files()` can be passed a non-package module since Python 3.12 (or importlib-resources 5.10.0, see https://importlib-resources.readthedocs.io/en/latest/history.html#v5-10-0 ), so we can use `__name__` instead of `__package__`. Also, I noticed that in lib/galaxy/config/__init__.py we are passing a directory to `importlib.resources.as_file()`, which is supported only since Python 3.12 (or importlib-resources 5.9.0). For the reasons above: - Update conditional imports in `lib/galaxy/util/resources.py` to import from `importlib.resources` only for Python 3.12, otherwise from the importlib-resources package. - Update importlib-resources requirement in galaxy-util package. - Drop importlib-resources requirement from galaxy-web-apps as it's not imported directly from its code any more. --- lib/galaxy/authnz/managers.py | 2 +- lib/galaxy/config/__init__.py | 2 +- lib/galaxy/config/templates.py | 2 +- lib/galaxy/exceptions/error_codes.py | 2 +- .../files/templates/examples/__init__.py | 2 +- lib/galaxy/managers/licenses.py | 2 +- lib/galaxy/managers/markdown_util.py | 2 +- lib/galaxy/navigation/data.py | 2 +- lib/galaxy/objectstore/examples/__init__.py | 2 +- .../templates/examples/__init__.py | 2 +- lib/galaxy/tool_util/client/landing.py | 2 +- lib/galaxy/tool_util/linters/datatypes.py | 2 +- .../tool_util/ontologies/ontology_data.py | 2 +- lib/galaxy/tool_util/upgrade/__init__.py | 2 +- lib/galaxy/util/resources.py | 22 +++++++++++-------- lib/galaxy/util/rules_dsl.py | 2 +- lib/galaxy/web/framework/base.py | 2 +- lib/galaxy/webapps/base/webapp.py | 2 +- lib/galaxy_test/base/populators.py | 6 ++--- lib/tool_shed/test/base/populators.py | 2 +- .../test/functional/test_shed_repositories.py | 2 +- packages/util/setup.cfg | 2 +- packages/web_apps/setup.cfg | 1 - .../tool_util/data/test_tool_data_bundles.py | 4 ++-- .../tool_util/test_parameter_specification.py | 4 ++-- 25 files changed, 40 insertions(+), 37 deletions(-) diff --git a/lib/galaxy/authnz/managers.py b/lib/galaxy/authnz/managers.py index 8a1145b9fd46..1d6ba0592e5f 100644 --- a/lib/galaxy/authnz/managers.py +++ b/lib/galaxy/authnz/managers.py @@ -26,7 +26,7 @@ PSAAuthnz, ) -OIDC_BACKEND_SCHEMA = resource_path(__package__, "xsd/oidc_backends_config.xsd") +OIDC_BACKEND_SCHEMA = resource_path(__name__, "xsd/oidc_backends_config.xsd") log = logging.getLogger(__name__) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index ef284bae1158..14bc863f19cc 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -70,7 +70,7 @@ ISO_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" GALAXY_APP_NAME = "galaxy" -GALAXY_SCHEMAS_PATH = resource_path(__package__, "schemas") +GALAXY_SCHEMAS_PATH = resource_path(__name__, "schemas") GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "config_schema.yml" REPORTS_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "reports_config_schema.yml" TOOL_SHED_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "tool_shed_config_schema.yml" diff --git a/lib/galaxy/config/templates.py b/lib/galaxy/config/templates.py index fc4b473ce7cc..04bf69adad14 100644 --- a/lib/galaxy/config/templates.py +++ b/lib/galaxy/config/templates.py @@ -33,7 +33,7 @@ def _get_template_body(template: str) -> str: def _get_template_path(relpath: str, custom_templates_dir: str) -> Traversable: """Return template file path.""" - default_path = resource_path("galaxy.config", "templates") / relpath + default_path = resource_path(__name__, "templates") / relpath custom_path = Path(custom_templates_dir) / relpath if custom_path.exists(): return custom_path diff --git a/lib/galaxy/exceptions/error_codes.py b/lib/galaxy/exceptions/error_codes.py index fe6c3e575177..0f978bcd4a24 100644 --- a/lib/galaxy/exceptions/error_codes.py +++ b/lib/galaxy/exceptions/error_codes.py @@ -43,7 +43,7 @@ def _from_dict(entry): return (name, ErrorCode(code, message)) -error_codes_json = resource_string(__package__, "error_codes.json") +error_codes_json = resource_string(__name__, "error_codes.json") error_codes_by_name: Dict[str, ErrorCode] = {} error_codes_by_int_code: Dict[int, ErrorCode] = {} diff --git a/lib/galaxy/files/templates/examples/__init__.py b/lib/galaxy/files/templates/examples/__init__.py index 3220cb7423f2..1bbf5b4b46fb 100644 --- a/lib/galaxy/files/templates/examples/__init__.py +++ b/lib/galaxy/files/templates/examples/__init__.py @@ -2,4 +2,4 @@ def get_example(filename) -> str: - return resource_string("galaxy.files.templates.examples", filename) + return resource_string(__name__, filename) diff --git a/lib/galaxy/managers/licenses.py b/lib/galaxy/managers/licenses.py index 47613aa739a8..f1c3b8f92585 100644 --- a/lib/galaxy/managers/licenses.py +++ b/lib/galaxy/managers/licenses.py @@ -65,7 +65,7 @@ class LicenseMetadataModel(BaseModel): "MPL-2.0", "PDDL-1.0", ] -SPDX_LICENSES_STRING = resource_string(__package__, "licenses.json") +SPDX_LICENSES_STRING = resource_string(__name__, "licenses.json") SPDX_LICENSES = json.loads(SPDX_LICENSES_STRING) for license in SPDX_LICENSES["licenses"]: license["recommended"] = license["licenseId"] in RECOMMENDED_LICENSES diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index ba32b6489624..be670bcc917c 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -767,7 +767,7 @@ def to_pdf_raw(basic_markdown: str, css_paths: Optional[List[str]] = None) -> by output_file.write(as_html) output_file.close() html = weasyprint.HTML(filename=index) - stylesheets = [weasyprint.CSS(string=resource_string(__package__, "markdown_export_base.css"))] + stylesheets = [weasyprint.CSS(string=resource_string(__name__, "markdown_export_base.css"))] for css_path in css_paths: with open(css_path) as f: css_content = f.read() diff --git a/lib/galaxy/navigation/data.py b/lib/galaxy/navigation/data.py index 31b8b0d3c9ed..9af996c2ac29 100644 --- a/lib/galaxy/navigation/data.py +++ b/lib/galaxy/navigation/data.py @@ -5,6 +5,6 @@ def load_root_component() -> Component: - new_data_yaml = resource_string(__package__, "navigation.yml") + new_data_yaml = resource_string(__name__, "navigation.yml") navigation_raw = yaml.safe_load(new_data_yaml) return Component.from_dict("root", navigation_raw) diff --git a/lib/galaxy/objectstore/examples/__init__.py b/lib/galaxy/objectstore/examples/__init__.py index 42b05b2d6eb2..ea099b6dcbcf 100644 --- a/lib/galaxy/objectstore/examples/__init__.py +++ b/lib/galaxy/objectstore/examples/__init__.py @@ -2,4 +2,4 @@ def get_example(filename: str) -> str: - return resource_string("galaxy.objectstore.examples", filename) + return resource_string(__name__, filename) diff --git a/lib/galaxy/objectstore/templates/examples/__init__.py b/lib/galaxy/objectstore/templates/examples/__init__.py index 2bcbd0645c79..1bbf5b4b46fb 100644 --- a/lib/galaxy/objectstore/templates/examples/__init__.py +++ b/lib/galaxy/objectstore/templates/examples/__init__.py @@ -2,4 +2,4 @@ def get_example(filename) -> str: - return resource_string("galaxy.objectstore.templates.examples", filename) + return resource_string(__name__, filename) diff --git a/lib/galaxy/tool_util/client/landing.py b/lib/galaxy/tool_util/client/landing.py index e089ae268689..63f5d7c7bc0f 100644 --- a/lib/galaxy/tool_util/client/landing.py +++ b/lib/galaxy/tool_util/client/landing.py @@ -24,7 +24,7 @@ def load_default_catalog(): - catalog_yaml = resource_string("galaxy.tool_util.client", "landing_catalog.sample.yml") + catalog_yaml = resource_string(__name__, "landing_catalog.sample.yml") return yaml.safe_load(catalog_yaml) diff --git a/lib/galaxy/tool_util/linters/datatypes.py b/lib/galaxy/tool_util/linters/datatypes.py index f05ae5c7ab23..63866cf0f84c 100644 --- a/lib/galaxy/tool_util/linters/datatypes.py +++ b/lib/galaxy/tool_util/linters/datatypes.py @@ -18,7 +18,7 @@ from galaxy.tool_util.parser import ToolSource from galaxy.util.resources import Traversable -DATATYPES_CONF = resource_path(__package__, "datatypes_conf.xml.sample") +DATATYPES_CONF = resource_path(__name__, "datatypes_conf.xml.sample") def _parse_datatypes(datatype_conf_path: Union[str, "Traversable"]) -> Set[str]: diff --git a/lib/galaxy/tool_util/ontologies/ontology_data.py b/lib/galaxy/tool_util/ontologies/ontology_data.py index 63a4d573b4d7..cb5f7b093965 100644 --- a/lib/galaxy/tool_util/ontologies/ontology_data.py +++ b/lib/galaxy/tool_util/ontologies/ontology_data.py @@ -27,7 +27,7 @@ def _multi_dict_mapping(content: str) -> Dict[str, List[str]]: def _read_ontology_data_text(filename: str) -> str: - return resource_string(__package__, filename) + return resource_string(__name__, filename) BIOTOOLS_MAPPING_FILENAME = "biotools_mappings.tsv" diff --git a/lib/galaxy/tool_util/upgrade/__init__.py b/lib/galaxy/tool_util/upgrade/__init__.py index 41be4988d0c4..299062f03b19 100644 --- a/lib/galaxy/tool_util/upgrade/__init__.py +++ b/lib/galaxy/tool_util/upgrade/__init__.py @@ -46,7 +46,7 @@ class AdviceCode(TypedDict): url: NotRequired[str] -upgrade_codes_json = resource_string(__package__, "upgrade_codes.json") +upgrade_codes_json = resource_string(__name__, "upgrade_codes.json") upgrade_codes_by_name: Dict[str, AdviceCode] = {} for name, upgrade_object in loads(upgrade_codes_json).items(): diff --git a/lib/galaxy/util/resources.py b/lib/galaxy/util/resources.py index f40a9b6071b3..60944e3f3ee8 100644 --- a/lib/galaxy/util/resources.py +++ b/lib/galaxy/util/resources.py @@ -3,17 +3,17 @@ import sys -if sys.version_info >= (3, 9): +if sys.version_info >= (3, 12): from importlib.resources import ( as_file, files, - Package as Anchor, ) + from importlib.resources.abc import Traversable - if sys.version_info >= (3, 12): - from importlib.resources.abc import Traversable + if sys.version_info >= (3, 13): + from importlib.resources import Anchor else: - from importlib.abc import Traversable + from importlib.resources import Package as Anchor else: from importlib_resources import ( as_file, @@ -23,20 +23,24 @@ from importlib_resources.abc import Traversable -def resource_path(package_or_requirement: Anchor, resource_name: str) -> Traversable: +def resource_path(anchor: Anchor, resource_name: str) -> Traversable: """ Return specified resource as a Traversable. + + anchor is either a module object or a module name as a string. """ - return files(package_or_requirement).joinpath(resource_name) + return files(anchor).joinpath(resource_name) -def resource_string(package_or_requirement: Anchor, resource_name: str) -> str: +def resource_string(anchor: Anchor, resource_name: str) -> str: """ Return specified resource as a string. Replacement function for pkg_resources.resource_string, but returns unicode string instead of bytestring. + + anchor is either a module object or a module name as a string. """ - return resource_path(package_or_requirement, resource_name).read_text() + return resource_path(anchor, resource_name).read_text() __all__ = ( diff --git a/lib/galaxy/util/rules_dsl.py b/lib/galaxy/util/rules_dsl.py index ebea78be82a1..83aca4560ec2 100644 --- a/lib/galaxy/util/rules_dsl.py +++ b/lib/galaxy/util/rules_dsl.py @@ -12,7 +12,7 @@ def get_rules_specification(): - return yaml.safe_load(resource_string(__package__, "rules_dsl_spec.yml")) + return yaml.safe_load(resource_string(__name__, "rules_dsl_spec.yml")) def _ensure_rule_contains_keys(rule, keys): diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 875494d3a378..dced82598fd6 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -35,7 +35,7 @@ #: time of the most recent server startup server_starttime = int(time.time()) try: - meta_json = json.loads(resource_string(__package__, "meta.json")) + meta_json = json.loads(resource_string(__name__, "meta.json")) server_starttime = meta_json.get("epoch") or server_starttime except Exception: meta_json = {} diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index f2be00448f20..611f4fa02c9a 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -181,7 +181,7 @@ def build_apispec(self): def create_mako_template_lookup(self, galaxy_app, name): paths = [] base_package = ( - "tool_shed.webapp" if galaxy_app.name == "tool_shed" else "galaxy.webapps.base" + "tool_shed.webapp" if galaxy_app.name == "tool_shed" else __name__ ) # reports has templates in galaxy package base_template_path = resource_path(base_package, "templates") with ExitStack() as stack: diff --git a/lib/galaxy_test/base/populators.py b/lib/galaxy_test/base/populators.py index 3f9a3043fd8a..ee32c27b822c 100644 --- a/lib/galaxy_test/base/populators.py +++ b/lib/galaxy_test/base/populators.py @@ -136,10 +136,10 @@ CWL_TOOL_DIRECTORY = os.path.join(galaxy_root_path, "test", "functional", "tools", "cwl_tools") # Simple workflow that takes an input and call cat wrapper on it. -workflow_str = resource_string(__package__, "data/test_workflow_1.ga") +workflow_str = resource_string(__name__, "data/test_workflow_1.ga") # Simple workflow that takes an input and filters with random lines twice in a # row - first grabbing 8 lines at random and then 6. -workflow_random_x2_str = resource_string(__package__, "data/test_workflow_2.ga") +workflow_random_x2_str = resource_string(__name__, "data/test_workflow_2.ga") DEFAULT_TIMEOUT = 60 # Secs to wait for state to turn ok @@ -1821,7 +1821,7 @@ def load_random_x2_workflow(self, name: str) -> dict: def load_workflow_from_resource(self, name: str, filename: Optional[str] = None) -> dict: if filename is None: filename = f"data/{name}.ga" - content = resource_string(__package__, filename) + content = resource_string(__name__, filename) return self.load_workflow(name, content=content) def simple_workflow(self, name: str, **create_kwds) -> str: diff --git a/lib/tool_shed/test/base/populators.py b/lib/tool_shed/test/base/populators.py index 642c1b76b9ac..7c2c31aba727 100644 --- a/lib/tool_shed/test/base/populators.py +++ b/lib/tool_shed/test/base/populators.py @@ -53,7 +53,7 @@ HasRepositoryId = Union[str, Repository] DEFAULT_PREFIX = "repofortest" -TEST_DATA_REPO_FILES = resource_path(__package__, "../test_data") +TEST_DATA_REPO_FILES = resource_path(__name__, "../test_data") COLUMN_MAKER_PATH = TEST_DATA_REPO_FILES.joinpath("column_maker/column_maker.tar") COLUMN_MAKER_1_1_1_PATH = TEST_DATA_REPO_FILES.joinpath("column_maker/column_maker_1.1.1.tar") DEFAULT_COMMIT_MESSAGE = "a test commit message" diff --git a/lib/tool_shed/test/functional/test_shed_repositories.py b/lib/tool_shed/test/functional/test_shed_repositories.py index bd4e88d26567..6d7cd61262c6 100644 --- a/lib/tool_shed/test/functional/test_shed_repositories.py +++ b/lib/tool_shed/test/functional/test_shed_repositories.py @@ -20,7 +20,7 @@ skip_if_api_v2, ) -COLUMN_MAKER_PATH = resource_path(__package__, "../test_data/column_maker/column_maker.tar") +COLUMN_MAKER_PATH = resource_path(__name__, "../test_data/column_maker/column_maker.tar") # test_0000 tests commit_message - find a way to test it here diff --git a/packages/util/setup.cfg b/packages/util/setup.cfg index 808606b77042..b52765c23c45 100644 --- a/packages/util/setup.cfg +++ b/packages/util/setup.cfg @@ -36,7 +36,7 @@ install_requires = bleach boltons docutils!=0.17,!=0.17.1 - importlib-resources;python_version<'3.9' + importlib-resources>=5.10.0;python_version<'3.12' packaging pyparsing PyYAML diff --git a/packages/web_apps/setup.cfg b/packages/web_apps/setup.cfg index 19ddb35ae0b3..fceb66451cc4 100644 --- a/packages/web_apps/setup.cfg +++ b/packages/web_apps/setup.cfg @@ -46,7 +46,6 @@ install_requires = fastapi>=0.101.0 gunicorn gxformat2 - importlib-resources;python_version<'3.9' Mako MarkupSafe Paste diff --git a/test/unit/tool_util/data/test_tool_data_bundles.py b/test/unit/tool_util/data/test_tool_data_bundles.py index 616323946a7f..b449178b316c 100644 --- a/test/unit/tool_util/data/test_tool_data_bundles.py +++ b/test/unit/tool_util/data/test_tool_data_bundles.py @@ -55,7 +55,7 @@ def test_xml_parsing() -> None: def test_parsing_manual() -> None: - with as_file(resource_path(__package__, "example_data_managers/manual.xml")) as path: + with as_file(resource_path(__name__, "example_data_managers/manual.xml")) as path: tree = parse_xml(path) data_managers_el = tree.getroot() data_manager_el = data_managers_el.find("data_manager") @@ -66,7 +66,7 @@ def test_parsing_manual() -> None: def test_parsing_mothur() -> None: - with as_file(resource_path(__package__, "example_data_managers/mothur.xml")) as path: + with as_file(resource_path(__name__, "example_data_managers/mothur.xml")) as path: tree = parse_xml(path) data_managers_el = tree.getroot() data_manager_el = data_managers_el.find("data_manager") diff --git a/test/unit/tool_util/test_parameter_specification.py b/test/unit/tool_util/test_parameter_specification.py index ed4b437449b5..41ec1fc9a95a 100644 --- a/test/unit/tool_util/test_parameter_specification.py +++ b/test/unit/tool_util/test_parameter_specification.py @@ -41,7 +41,7 @@ def specification_object(): try: - yaml_str = resource_string(__package__, "parameter_specification.yml") + yaml_str = resource_string(__name__, "parameter_specification.yml") except AttributeError: # hack for the main() function below where this file is interpreted as part of the # Galaxy tree. @@ -54,7 +54,7 @@ def framework_tool_checks(): # There is something beautiful about a targeted tool for every parameter feature but realistically # we've been doing a version of the for a decade with tool tests and we can leverage those also. try: - yaml_str = resource_string(__package__, "framework_tool_checks.yml") + yaml_str = resource_string(__name__, "framework_tool_checks.yml") except AttributeError: # hack for the main() function below where this file is interpreted as part of the # Galaxy tree.