Don't install Galaxy packages in editable mode when testing them

When enabling the `warn_unused_ignores = True` option of mypy, this
was making the `test_galaxy_packages` tests fail with:

```
mypy .
Traceback (most recent call last):
  File "/tmp/gxpkgtestenvu3fgDr/bin/mypy", line 8, in <module>
    sys.exit(console_entry())
  File "/tmp/gxpkgtestenvu3fgDr/lib/python3.10/site-packages/mypy/__main__.py", line 15, in console_entry
    main()
  File "mypy/main.py", line 100, in main
  File "mypy/main.py", line 182, in run_build
  File "mypy/build.py", line 192, in build
  File "mypy/build.py", line 266, in _build
  File "mypy/build.py", line 2942, in dispatch
  File "mypy/build.py", line 3333, in process_graph
  File "mypy/build.py", line 3414, in process_fresh_modules
  File "mypy/build.py", line 2110, in fix_cross_refs
  File "mypy/fixup.py", line 52, in fixup_module
  File "mypy/fixup.py", line 127, in visit_symbol_table
  File "mypy/lookup.py", line 49, in lookup_fully_qualified
AssertionError: Cannot find component 'util' for 'galaxy.util'
make: *** [Makefile:96: mypy] Error 1
```

Moreover, installing packages in editable mode was hiding some issues
with how files from other packages were accessed.

lib/galaxy/authnz/managers.py

@@ -22,7 +22,10 @@
     string_as_bool,
     unicodify,
 )
-from galaxy.util.resources import files
+from galaxy.util.resources import (
+    as_file,
+    resource_path,
+)
 from .custos_authnz import (
     CustosAuthFactory,
     KEYCLOAK_BACKENDS,
@@ -35,7 +38,7 @@
     Strategy,
 )
 
-OIDC_BACKEND_SCHEMA = files("galaxy.authnz.xsd") / "oidc_backends_config.xsd"
+OIDC_BACKEND_SCHEMA = resource_path(__package__, "xsd/oidc_backends_config.xsd")
 
 log = logging.getLogger(__name__)
 
@@ -108,7 +111,8 @@ def _parse_oidc_backends_config(self, config_file):
         self.oidc_backends_config = {}
         self.oidc_backends_implementation = {}
         try:
-            tree = parse_xml(config_file, schemafname=OIDC_BACKEND_SCHEMA)
+            with as_file(OIDC_BACKEND_SCHEMA) as oidc_backend_schema_path:
+                tree = parse_xml(config_file, schemafname=oidc_backend_schema_path)
             root = tree.getroot()
             if root.tag != "OIDC":
                 raise etree.ParseError(

lib/galaxy/config/__init__.py

@@ -51,7 +51,10 @@
     read_properties_from_file,
     running_from_source,
 )
-from galaxy.util.resources import files
+from galaxy.util.resources import (
+    as_file,
+    resource_path,
+)
 from galaxy.util.themes import flatten_theme
 from ..version import (
     VERSION_MAJOR,
@@ -67,7 +70,7 @@
 ISO_DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
 
 GALAXY_APP_NAME = "galaxy"
-GALAXY_SCHEMAS_PATH = files("galaxy.config") / "schemas"
+GALAXY_SCHEMAS_PATH = resource_path(__package__, "schemas")
 GALAXY_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "config_schema.yml"
 REPORTS_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "reports_config_schema.yml"
 TOOL_SHED_CONFIG_SCHEMA_PATH = GALAXY_SCHEMAS_PATH / "tool_shed_config_schema.yml"
@@ -189,7 +192,7 @@ def configure_logging(config, facts=None):
         logging.config.dictConfig(logging_conf)
 
 
-def find_root(kwargs):
+def find_root(kwargs) -> str:
     return os.path.abspath(kwargs.get("root_dir", "."))
 
 
@@ -234,6 +237,7 @@ class BaseAppConfiguration(HasDynamicProperties):
     add_sample_file_to_defaults: Set[str] = set()  # for these options, add sample config files to their defaults
     listify_options: Set[str] = set()  # values for these options are processed as lists of values
     object_store_store_by: str
+    shed_tools_dir: str
 
     def __init__(self, **kwargs):
         self._preprocess_kwargs(kwargs)
@@ -831,7 +835,8 @@ def _process_config(self, kwargs: Dict[str, Any]) -> None:
         self.cookie_path = kwargs.get("cookie_path")
         if not running_from_source and kwargs.get("tool_path") is None:
             try:
-                self.tool_path = str(files("galaxy.tools") / "bundled")
+                with as_file(resource_path("galaxy.tools", "bundled")) as path:
+                    self.tool_path = os.fspath(path)
             except ModuleNotFoundError:
                 # Might not be a full galaxy installation
                 self.tool_path = self._in_root_dir(self.tool_path)

lib/galaxy/config/config_manage.py

@@ -46,6 +46,7 @@
     nice_config_parser,
     NicerConfigParser,
 )
+from galaxy.util.resources import Traversable
 from galaxy.util.yaml_util import (
     ordered_dump,
     ordered_load,
@@ -72,7 +73,7 @@ class App(NamedTuple):
     default_port: str
     expected_app_factories: List[str]
     destination: str
-    schema_path: str
+    schema_path: Traversable
 
     @property
     def app_name(self) -> str:
@@ -219,21 +220,21 @@ class OptionValue(NamedTuple):
     "8080",
     ["galaxy.web.buildapp:app_factory"],
     "config/galaxy.yml",
-    str(GALAXY_CONFIG_SCHEMA_PATH),
+    GALAXY_CONFIG_SCHEMA_PATH,
 )
 SHED_APP = App(
     ["tool_shed_wsgi.ini", "config/tool_shed.ini"],
     "9009",
     ["tool_shed.webapp.buildapp:app_factory"],
     "config/tool_shed.yml",
-    str(TOOL_SHED_CONFIG_SCHEMA_PATH),
+    TOOL_SHED_CONFIG_SCHEMA_PATH,
 )
 REPORTS_APP = App(
     ["reports_wsgi.ini", "config/reports.ini"],
     "9001",
     ["galaxy.webapps.reports.buildapp:app_factory"],
     "config/reports.yml",
-    str(REPORTS_CONFIG_SCHEMA_PATH),
+    REPORTS_CONFIG_SCHEMA_PATH,
 )
 APPS = {"galaxy": GALAXY_APP, "tool_shed": SHED_APP, "reports": REPORTS_APP}
 

lib/galaxy/config/schema.py

@@ -1,6 +1,7 @@
 import logging
 
 from galaxy.exceptions import ConfigurationError
+from galaxy.util.resources import Traversable
 from galaxy.util.yaml_util import ordered_load
 
 log = logging.getLogger(__name__)
@@ -35,15 +36,15 @@ def get_app_option(self, name):
 
 
 class AppSchema(Schema):
-    def __init__(self, schema_path, app_name):
+    def __init__(self, schema_path: Traversable, app_name: str):
         self.raw_schema = self._read_schema(schema_path)
         self.description = self.raw_schema.get("desc", None)
         app_schema = self.raw_schema["mapping"][app_name]["mapping"]
         self._preprocess(app_schema)
         super().__init__(app_schema)
 
-    def _read_schema(self, path):
-        with open(path) as f:
+    def _read_schema(self, path: Traversable):
+        with path.open() as f:
             return ordered_load(f)
 
     def _preprocess(self, app_schema):

lib/galaxy/datatypes/binary.py

@@ -4367,7 +4367,7 @@ class HexrdImagesNpz(Npz):
     >>> fname = get_test_fname('hexrd.images.npz')
     >>> HexrdImagesNpz().sniff(fname)
     True
-    >>> fname = get_test_fname('eta_ome.npz')
+    >>> fname = get_test_fname('hexrd.eta_ome.npz')
     >>> HexrdImagesNpz().sniff(fname)
     False
     """

lib/galaxy/datatypes/media.py

@@ -193,8 +193,6 @@ class Mp4(Video):
     >>> from galaxy.datatypes.sniff import sniff_with_cls
     >>> sniff_with_cls(Mp4, 'video_1.mp4')
     True
-    >>> sniff_with_cls(Mp4, 'audio_1.mp4')
-    False
     """
 
     file_ext = "mp4"

lib/galaxy/datatypes/sniff.py

@@ -49,8 +49,9 @@
 
 def get_test_fname(fname):
     """Returns test data filename"""
-    path, name = os.path.split(__file__)
+    path = os.path.dirname(__file__)
     full_path = os.path.join(path, "test", fname)
+    assert os.path.isfile(full_path), f"{full_path} is not a file"
     return full_path
 
 

lib/galaxy/tool_util/ontologies/ontology_data.py

@@ -9,7 +9,7 @@
 
 from galaxy.tool_util.biotools import BiotoolsMetadataSource
 from galaxy.tool_util.parser import ToolSource
-from galaxy.util.resources import files
+from galaxy.util.resources import resource_string
 
 
 def _multi_dict_mapping(content: str) -> Dict[str, List[str]]:
@@ -23,10 +23,9 @@ def _multi_dict_mapping(content: str) -> Dict[str, List[str]]:
 
 
 def _read_ontology_data_text(filename: str) -> str:
-    return files(PACKAGE).joinpath(filename).read_text()
+    return resource_string(__package__, filename)
 
 
-PACKAGE = "galaxy.tool_util.ontologies"
 BIOTOOLS_MAPPING_FILENAME = "biotools_mappings.tsv"
 EDAM_OPERATION_MAPPING_FILENAME = "edam_operation_mappings.tsv"
 EDAM_TOPIC_MAPPING_FILENAME = "edam_topic_mappings.tsv"

lib/galaxy/util/__init__.py

@@ -34,6 +34,7 @@
 from email.mime.text import MIMEText
 from hashlib import md5
 from os.path import relpath
+from pathlib import Path
 from typing import (
     Any,
     cast,
@@ -1736,25 +1737,23 @@ def safe_str_cmp(a, b):
     return rv == 0
 
 
-#  Don't use these two directly, prefer method version that "works" with packaged Galaxy.
-galaxy_root_path = os.path.join(__path__[0], os.pardir, os.pardir, os.pardir)  # type: ignore[name-defined]
-galaxy_samples_path = os.path.join(__path__[0], os.pardir, "config", "sample")  # type: ignore[name-defined]
+#  Don't use this directly, prefer method version that "works" with packaged Galaxy.
+galaxy_root_path = Path(__file__).parent.parent.parent.parent
 
 
-def galaxy_directory():
-    path = galaxy_root_path
+def galaxy_directory() -> str:
     if in_packages():
-        path = os.path.join(galaxy_root_path, "..")
+        # This will work only when running pytest from <galaxy_root>/packages/<package_name>/
+        cwd = Path.cwd()
+        path = cwd.parent.parent
+    else:
+        path = galaxy_root_path
     return os.path.abspath(path)
 
 
-def in_packages():
-    # Normalize first; otherwise basename will be `..`
-    return os.path.basename(os.path.normpath(galaxy_root_path)) == "packages"
-
-
-def galaxy_samples_directory():
-    return os.path.join(galaxy_directory(), "lib", "galaxy", "config", "sample")
+def in_packages() -> bool:
+    galaxy_lib_path = Path(__file__).parent.parent.parent
+    return galaxy_lib_path.name != "lib"
 
 
 def config_directories_from_setting(directories_setting, galaxy_root=galaxy_root_path):

lib/galaxy/webapps/base/webapp.py

@@ -8,6 +8,7 @@
 import re
 import socket
 import time
+from contextlib import ExitStack
 from http.cookies import CookieError
 from typing import (
     Any,
@@ -51,7 +52,10 @@
     safe_makedirs,
     unicodify,
 )
-from galaxy.util.resources import files
+from galaxy.util.resources import (
+    as_file,
+    resource_path,
+)
 from galaxy.util.sanitize_html import sanitize_html
 from galaxy.version import VERSION
 from galaxy.web.framework import (
@@ -178,16 +182,19 @@ def create_mako_template_lookup(self, galaxy_app, name):
         base_package = (
             "tool_shed.webapp" if galaxy_app.name == "tool_shed" else "galaxy.webapps.base"
         )  # reports has templates in galaxy package
-        base_template_path = files(base_package) / "templates"
-        # First look in webapp specific directory
-        if name is not None:
-            paths.append(base_template_path / "webapps" / name)
-        # Then look in root directory
-        paths.append(base_template_path)
-        # Create TemplateLookup with a small cache
-        return mako.lookup.TemplateLookup(
-            directories=paths, module_directory=galaxy_app.config.template_cache_path, collection_size=500
-        )
+        base_template_path = resource_path(base_package, "templates")
+        with ExitStack() as stack:
+            # First look in webapp specific directory
+            if name is not None:
+                path = stack.enter_context(as_file(base_template_path / "webapps" / name))
+                paths.append(path)
+            # Then look in root directory
+            path = stack.enter_context(as_file(base_template_path))
+            paths.append(path)
+            # Create TemplateLookup with a small cache
+            return mako.lookup.TemplateLookup(
+                directories=paths, module_directory=galaxy_app.config.template_cache_path, collection_size=500
+            )
 
     def handle_controller_exception(self, e, trans, method, **kwargs):
         if isinstance(e, TypeError):

lib/tool_shed/test/base/populators.py

@@ -2,6 +2,7 @@
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import (
+    Iterator,
     List,
     Optional,
     Union,
@@ -11,7 +12,7 @@
 from typing_extensions import Protocol
 
 from galaxy.util.resources import (
-    files,
+    as_file,
     resource_path,
     Traversable,
 )
@@ -50,31 +51,27 @@
 HasRepositoryId = Union[str, Repository]
 
 DEFAULT_PREFIX = "repofortest"
-COLUMN_MAKER_PATH = resource_path(__package__, "../test_data/column_maker/column_maker.tar")
-COLUMN_MAKER_1_1_1_PATH = resource_path(__package__, "../test_data/column_maker/column_maker.tar")
+TEST_DATA_REPO_FILES = resource_path(__package__, "../test_data")
+COLUMN_MAKER_PATH = TEST_DATA_REPO_FILES.joinpath("column_maker/column_maker.tar")
+COLUMN_MAKER_1_1_1_PATH = TEST_DATA_REPO_FILES.joinpath("column_maker/column_maker_1.1.1.tar")
 DEFAULT_COMMIT_MESSAGE = "a test commit message"
-TEST_DATA_REPO_FILES = files("tool_shed.test.test_data")
 
 
-def repo_files(test_data_path: str) -> List[Path]:
+def repo_files(test_data_path: str) -> Iterator[Path]:
     repos = TEST_DATA_REPO_FILES.joinpath(f"repos/{test_data_path}")
-    paths = sorted(Path(str(x)) for x in repos.iterdir())
-    return paths
+    for child in sorted(_.name for _ in repos.iterdir()):
+        with as_file(repos.joinpath(child)) as path:
+            yield path
 
 
-def repo_tars(test_data_path: str) -> List[Path]:
-    tar_paths = []
+def repo_tars(test_data_path: str) -> Iterator[Path]:
     for path in repo_files(test_data_path):
-        if path.is_dir():
-            prefix = f"shedtest_{test_data_path}_{path.name}_"
-            tf = NamedTemporaryFile(delete=False, prefix=prefix)
+        assert path.is_dir()
+        prefix = f"shedtest_{test_data_path}_{path.name}_"
+        with NamedTemporaryFile(prefix=prefix) as tf:
             with tarfile.open(tf.name, "w:gz") as tar:
                 tar.add(str(path.absolute()), arcname=test_data_path or path.name)
-            tar_path = tf.name
-        else:
-            tar_path = str(path)
-        tar_paths.append(Path(tar_path))
-    return tar_paths
+            yield Path(tf.name)
 
 
 class HostsTestToolShed(Protocol):

mypy.ini

@@ -4,9 +4,10 @@ show_error_codes = True
 ignore_missing_imports = True
 check_untyped_defs = True
 exclude = (?x)(
-        ^lib/galaxy/tools/bundled
-        | ^test/functional
-        | .*tool_shed/test/test_data/repos
+        ^build/
+        | ^lib/galaxy/tools/bundled/
+        | ^test/functional/
+        | .*tool_shed/test/test_data/repos/
     )
 pretty = True
 no_implicit_reexport = True

packages/config/MANIFEST.in

@@ -1,3 +1,4 @@
 include *.rst *.txt LICENSE */py.typed
 include galaxy/config/schemas/*.yml
 include galaxy/config/sample/*.sample*
+graft galaxy/config/templates/

packages/data/setup.cfg

@@ -34,6 +34,7 @@ include_package_data = True
 install_requires =
     galaxy-files
     galaxy-objectstore
+    galaxy-schema
     galaxy-tool-util
     galaxy-util[template]
     alembic

packages/test.sh

@@ -49,11 +49,11 @@ while read -r package_dir || [ -n "$package_dir" ]; do  # https://stackoverflow.
 
     # Install extras (if needed)
     if [ "$package_dir" = "util" ]; then
-        pip install -e '.[template,jstree]'
+        pip install '.[template,jstree]'
     elif [ "$package_dir" = "tool_util" ]; then
-        pip install -e '.[cwl,mulled,edam,extended-assertions]'
+        pip install '.[cwl,mulled,edam,extended-assertions]'
     else
-        pip install -e '.'
+        pip install .
     fi
 
     pip install -r test-requirements.txt