diff --git a/hab/__init__.py b/hab/__init__.py index fa27b9d..2fac716 100644 --- a/hab/__init__.py +++ b/hab/__init__.py @@ -1,10 +1,10 @@ -__all__ = ["__version__", "NotSet", "Resolver", "Site"] +__all__ = ["__version__", "DistroMode", "NotSet", "Resolver", "Site"] from .utils import NotSet # Note: Future imports depend on NotSet so it must be imported first # isort: split -from .resolver import Resolver +from .resolver import DistroMode, Resolver from .site import Site from .version import version as __version__ diff --git a/hab/cache.py b/hab/cache.py index bf1f6ca..51cc003 100644 --- a/hab/cache.py +++ b/hab/cache.py @@ -111,6 +111,7 @@ def generate_cache(self, resolver, site_file, version=1): the provided site file. Use this method any time changes are made that hab needs to be aware of. Caching is enabled by the existence of this file. """ + from .distro_finders.distro_finder import DistroFinder from .site import Site # Indicate the version specification this habcache file conforms to. @@ -125,6 +126,9 @@ def generate_cache(self, resolver, site_file, version=1): glob_str, cls = stats # Process each glob dir defined for this site for dirname in temp_site.get(key, []): + # Caching is only supported for direct file paths + if isinstance(dirname, DistroFinder): + dirname = dirname.root cfg_paths = output.setdefault(key, {}).setdefault( platform_path_key(dirname).as_posix(), {} ) @@ -180,7 +184,7 @@ def iter_cache_paths(cls, name, paths, cache, glob_str=None, include_path=True): logger.debug(f"Using glob for {name} dir: {dirname}") # Fallback to globing the file system if glob_str: - paths = sorted(glob.glob(str(dirname / glob_str))) + paths = utils.glob_path(dirname / glob_str) else: paths = [] if not include_path: diff --git a/hab/cli.py b/hab/cli.py index 579ae43..63160dd 100644 --- a/hab/cli.py +++ b/hab/cli.py @@ -9,7 +9,7 @@ from click.shell_completion import CompletionItem from colorama import Fore -from . import Resolver, Site, __version__, utils +from . import DistroMode, Resolver, Site, __version__, utils from .parsers.unfrozen_config import UnfrozenConfig logger = logging.getLogger(__name__) @@ -603,7 +603,18 @@ def env(settings, uri, launch): "--type", "report_type", type=click.Choice( - ["nice", "site", "s", "uris", "u", "versions", "v", "forest", "f", "all-uris"] + # Note: Put short names on same line as full name + # fmt: off + [ + "nice", + "site", "s", + "uris", "u", + "versions", "v", + "downloads", + "forest", "f", + "all-uris", + ] + # fmt: on ), default="nice", help="Type of report.", @@ -644,7 +655,7 @@ def dump(settings, uri, env, env_config, report_type, flat, verbosity, format_ty resolver = settings.resolver - if report_type in ("uris", "versions", "forest"): + if report_type in ("uris", "versions", "downloads", "forest"): from .parsers.format_parser import FormatParser formatter = FormatParser(verbosity, color=True) @@ -659,16 +670,22 @@ def dump(settings, uri, env, env_config, report_type, flat, verbosity, format_ty resolver.configs, fmt=formatter.format ): click.echo(line) - if report_type in ("versions", "forest"): + if report_type in ("versions", "downloads", "forest"): click.echo(f'{Fore.YELLOW}{" Versions ".center(50, "-")}{Fore.RESET}') - for line in resolver.dump_forest( - resolver.distros, - attr="name", - fmt=formatter.format, - truncate=truncate, - ): - click.echo(line) + mode = ( + DistroMode.Downloaded + if report_type == "downloads" + else DistroMode.Installed + ) + with resolver.distro_mode_override(mode): + for line in resolver.dump_forest( + resolver.distros, + attr="name", + fmt=formatter.format, + truncate=truncate, + ): + click.echo(line) elif report_type == "all-uris": # Combines all non-placeholder URI's into a single json document and display. # This can be used to compare changes to configs when editing them in bulk. @@ -786,6 +803,59 @@ def cache(settings, path): click.echo(f"Cache took: {e - s}, cache file: {out}") +@_cli.command() +@click.option( + "-u", + "--uri", + "uris", + multiple=True, + help="A URI that is resolved and all required distros are installed. Can " + "be used multiple times and each URI's distros are resolved independently.", +) +@click.option( + "-d", + "--distro", + "distros", + multiple=True, + help="Additional distros to install. Can be used multiple times and each use " + "is resolved independently.", +) +@click.option( + "--dry-run/--no-dry-run", + default=False, + help="Don't actually install anything, just print what would be installed.", +) +@click.option( + "--force-reinstall/--no-force-reinstall", + default=False, + help="Reinstall all resolved distros even if they are already installed.", +) +@click.option( + "--target", + type=click.Path(file_okay=False, resolve_path=True), + help="Install distros into DIRECTORY. Defaults to the sites " + 'downloads["install_root"] setting.', +) +@click.pass_obj +def install(settings, uris, distros, dry_run, force_reinstall, target): + """Install distros for use in hab. At least one uri or distro must be + specified to install. This is intended to install all versions of hab distros + that are required for a collection of hab URI on this system. This means that + unlike pip this may install multiple versions of hab distros. + """ + distros = list(distros) if distros else None + uris = list(uris) if uris else None + if not distros and not uris: + raise ValueError("You must specify at least one --uri or --distro to install.") + settings.resolver.install( + uris=uris, + additional_distros=distros, + target=target, + dry_run=dry_run, + replace=force_reinstall, + ) + + def cli(*args, **kwargs): """Runs the hab cli. If an exception is raised, only the exception message is printed and the stack trace is hidden. Use `hab -v ...` to enable showing diff --git a/hab/resolver.py b/hab/resolver.py index a8f1b10..6a6f3e0 100644 --- a/hab/resolver.py +++ b/hab/resolver.py @@ -1,14 +1,17 @@ # __all__ = ["Resolver"] +import concurrent.futures import copy +import enum import logging +from contextlib import contextmanager import anytree from packaging.requirements import Requirement from . import utils -from .errors import _IgnoredVersionError -from .parsers import Config, DistroVersion, HabBase +from .errors import HabError, _IgnoredVersionError +from .parsers import Config, HabBase from .site import Site from .solvers import Solver from .user_prefs import UserPrefs @@ -16,6 +19,17 @@ logger = logging.getLogger(__name__) +class DistroMode(enum.Enum): + """Used by `hab.Revolver` to control which forest is used to resolve distros.""" + + # TODO: Switch docstrings to `Annotated` if we move to py 3.9+ only + # support https://stackoverflow.com/a/78361486 + Downloaded = enum.auto() + """Use the `downloadable_distros` forest when using `Resolver.distros`.""" + Installed = enum.auto() + """Use the `installed_distros` forest when using `Resolver.distros`.""" + + class Resolver(object): """Used to resolve a hab environment setup and apply it to the current environment. @@ -67,7 +81,9 @@ def __init__( logger.debug(f"distro_paths: {self.distro_paths}") self._configs = None - self._distros = None + self.distro_mode = DistroMode.Installed + self._downloadable_distros = None + self._installed_distros = None self.ignored = self.site["ignored_distros"] # If true, then all scripts are printed instead of being written to disk @@ -80,8 +96,10 @@ def clear_caches(self): """Clears cached resolved data so it is re-generated on next use.""" logger.debug("Resolver cache cleared.") self._configs = None - self._distros = None + self._downloadable_distros = None + self._installed_distros = None self.site.cache.clear() + [distro_finder.clear_cache() for distro_finder in self.distro_paths] def closest_config(self, path, default=False): """Returns the most specific leaf or the tree root matching path. Ignoring any @@ -165,8 +183,33 @@ def configs(self): self._configs = self.parse_configs(self.config_paths) return self._configs + @contextmanager + def distro_mode_override(self, mode): + """A context manager that sets `distro_mode` while inside the context. + This lets you switch which distro forest is returned by the `distro` method. + + Example: + + assert resolver.distros == resolver.installed_distros + with resolver.distro_mode_override(DistroMode.Downloaded): + assert resolver.distros == resolver.downloadable_distros + assert resolver.distros == resolver.installed_distros + """ + if not isinstance(mode, DistroMode): + raise ValueError("You can only specify DistroModes.") + + current = self.distro_mode + logger.debug(f"Setting Resolver distro_mode to {mode} from {current}.") + try: + self.distro_mode = mode + yield current + finally: + self.distro_mode = current + logger.debug(f"Restored distro_mode to {self.distro_mode}.") + @property def distro_paths(self): + """`DistroFinder`s used to populate `installed_distros`.""" return self.site["distro_paths"] @distro_paths.setter @@ -176,15 +219,46 @@ def distro_paths(self, paths): paths = utils.Platform.expand_paths(paths) self.site["distro_paths"] = paths - # Reset _distros so we re-generate them the next time they are requested - self._distros = None + # Reset the cache so we re-generate it the next time it is requested. + self._installed_distros = None + + @property + def installed_distros(self): + """A dictionary of all usable distros that have been parsed for this resolver. + + These are the distros used by hab when a hab environment is configured + and aliases(programs) access these files. + """ + if self._installed_distros is None: + self._installed_distros = self.parse_distros(self.distro_paths) + return self._installed_distros @property def distros(self): - """A list of all of the requested distros to resolve.""" - if self._distros is None: - self._distros = self.parse_distros(self.distro_paths) - return self._distros + """A dictionary of distros for this resolver. + + This forest is used to resolve distro dependencies into config versions. + + The output is dependent on `self.distro_mode`, use the `distro_mode_override` + context manager to change the mode temporarily. + """ + if self.distro_mode == DistroMode.Downloaded: + return self.downloadable_distros + return self.installed_distros + + @property + def downloadable_distros(self): + """A dictionary of all distros that can be installed into `installed_distros`. + + This is used by the hab install process, not when enabling a hab + environment. These distros are available to download and install for use + in the `installed_distros` forest. + """ + if self._downloadable_distros is None: + self._downloadable_distros = self.parse_distros( + self.site.downloads["distros"] + ) + return self._downloadable_distros @classmethod def dump_forest( @@ -291,6 +365,98 @@ def freeze_configs(self): out[uri] = cfg.freeze() return out + def install( + self, + uris=None, + additional_distros=None, + target=None, + dry_run=True, + replace=False, + ): + """Ensure the required distros are installed for use in hab. + + Resolves the distros defined by one or more URI's and additional distros + against the distro versions available on from a `downloadable_distros`. + Then extracts them into a target location for use in hab environments. + + Each URI and additional_distros requirement is resolved independently. This + allows you to install multiple versions of a given distro so the correct + one is available when a given URI is used in a hab environment. + + Args: + uris (list, optional): A list of URI strings. These URI's are resolved + against the available distros in `downloadable_distros`. + additional_distros (list, optional): A list of additional distro + requirements to resolve and install. + target (os.PathLike, optional): The target directory to install all + resolved distros into. This is the root directory. The per-distro + name and version paths are added relative to this directory based + on the `site.downloads["relative_path"]` setting. + dry_run (bool, optional): If True then don't actually install the + the distros. The returned list is the final list of all distros + that need to be installed. + replace (bool, optional): This method skips installing any distros + that are already installed. Setting this to True will delete the + existing distros before re-installing them. + + Returns: + list: A list of DistroVersion's that were installed. The distros are + from `downloadable_distros` and represent the remote resources. + """ + + def str_distros(distros, sep=", "): + return sep.join(sorted([d.name for d in missing])) + + if target is None: + target = self.site.downloads.get("install_root") + if target is None: + raise HabError( + 'You must specify target, or set ["downloads"]["install_root"] ' + "in your site config." + ) + + distros = set() + if uris is None: + uris = [] + # Resolve all distros and any additional distros they may require from + # the download forest. + with self.distro_mode_override(DistroMode.Downloaded): + if additional_distros: + requirements = self.resolve_requirements(additional_distros) + for req in requirements.values(): + version = self.find_distro(req) + distros.add(version) + + for uri in uris: + cfg = self.resolve(uri) + distros.update(cfg.versions) + + # Check the installed forest for any that are not already installed and + # build a list of missing distros. + missing = [] + for distro in distros: + if replace: + missing.append(distro) + continue + + installed = self.find_distro(distro.distro_name) + if not installed: + missing.append(distro) + + if dry_run: + logger.warning(f"Dry Run would install distros: {str_distros(missing)}") + return missing + + # Download and install the missing distros using threading to speed up + # the download process. + logger.warning(f"Installing distros: {str_distros(missing)}") + with concurrent.futures.ThreadPoolExecutor() as executor: + for distro in missing: + logger.warning(f"Installing distro: {distro.name}") + executor.submit(distro.install, target, replace=replace) + logger.warning(f"Installed distros: {str_distros(missing)}") + return missing + @classmethod def instance(cls, name="main", **kwargs): """Returns a shared Resolver instance for name, initializing it if required. @@ -325,14 +491,16 @@ def parse_configs(self, config_paths, forest=None): Config(forest, self, path, root_paths=set((dirname,))) return forest - def parse_distros(self, distro_paths, forest=None): + def parse_distros(self, distro_finders, forest=None): + """Parse all provided DistroFinders and populate the forest of distros.""" if forest is None: forest = {} - for dirname, path in self.site.distro_paths(distro_paths): - try: - DistroVersion(forest, self, path, root_paths=set((dirname,))) - except _IgnoredVersionError as error: - logger.debug(str(error)) + for distro_finder in distro_finders: + for _, path, _ in distro_finder.distro_path_info(): + try: + distro_finder.distro(forest, self, path) + except _IgnoredVersionError as error: + logger.debug(str(error)) return forest def resolve(self, uri, forced_requirements=None): diff --git a/hab/site.py b/hab/site.py index 7e10ab2..8593ff8 100644 --- a/hab/site.py +++ b/hab/site.py @@ -4,6 +4,7 @@ from pathlib import Path, PurePosixPath, PureWindowsPath from colorama import Fore, Style +from importlib_metadata import EntryPoint from . import utils from .cache import Cache @@ -39,6 +40,7 @@ def __init__(self, paths=None, platform=None): if platform is None: platform = utils.Platform.name() self.platform = platform + self._downloads_parsed = False # Add default data to all site instances. Site data is only valid for # the current platform, so discard any other platform configurations. @@ -64,6 +66,50 @@ def __init__(self, paths=None, platform=None): def data(self): return self.frozen_data.get(self.platform) + @property + def downloads(self): + """A dictionary of configuration information for downloading distros. + + The key "distros" should contain a list of `DistroFinder` instances similar + to "distro_paths". These are used to find and download distro versions. + + The key "cache_root" contains the Path to the directory where remote files + are downloaded for installation. + + The key "install_root" indicates where distros are installed. This should + normally be one of the "distro_paths" but should not contain glob wildcards. + """ + if self._downloads_parsed: + return self["downloads"] + + self._downloads_parsed = True + downloads = self.setdefault("downloads", {}) + + # Convert distros data into DistroFinder classes + distros = [] + for distro_finder in downloads.get("distros", []): + inst = self.entry_point_init( + "hab.download.finder", distro_finder[0], distro_finder[1:] + ) + # Ensure these items can access the site and its cache + inst.site = self + distros.append(inst) + downloads["distros"] = distros + + # Configure the download cache directory + cache_root = utils.Platform.default_download_cache() + if downloads.get("cache_root"): + # Use cache_root if its set to a non-empty value + paths = utils.Platform.expand_paths(downloads["cache_root"]) + if paths: + cache_root = paths[0] + downloads["cache_root"] = cache_root + + if "install_root" in downloads: + downloads["install_root"] = Path(downloads["install_root"]) + + return self["downloads"] + def dump(self, verbosity=0, color=None, width=80): """Return a string of the properties and their values. @@ -111,6 +157,9 @@ def cached_fmt(path, cached): {"HAB_PATHS": hab_paths}, color=color, width=width, verbosity=verbosity ) + # Ensure lazy loaded code is run before dumping + self.downloads + # Include all of the resolved site configurations ret = [] for prop, value in self.items(): @@ -134,6 +183,30 @@ def cached_fmt(path, cached): ret = "\n".join(ret) return utils.dump_title("Dump of Site", f"{site_ret}\n{ret}", color=color) + def entry_point_init(self, group, value, args, name=""): + """Initialize an entry point with args and kwargs. + + Args: + group (str): The entry point group name. + value (str): The entry point value used to import and resolve the class. + args (list): A list of arguments to pass to the class on init. If the + last item in this list is a dict, then it is passed to the kwargs + of the class. If not specified then the kwarg `site` will be set + to self. + name (str, optional): The entry point name. + + Returns: + A initialized object defined by the inputs. + """ + ep = EntryPoint(name, value, group) + ep_cls = ep.load() + kwargs = {} + if args and isinstance(args[-1], dict): + kwargs = args.pop() + if "site" not in kwargs: + kwargs["site"] = self + return ep_cls(*args, **kwargs) + def entry_points_for_group( self, group, default=None, entry_points=None, omit_none=True ): @@ -152,10 +225,6 @@ def entry_points_for_group( then don't include an EntryPoint object for it in the return. This allows a second site file to disable a entry_point already set. """ - # Delay this import to when required. It's faster than pkg_resources but - # no need to pay the import price for it if you are not using it. - from importlib_metadata import EntryPoint - ret = [] # Use the site defined entry_points if an override dict wasn't provided if entry_points is None: @@ -208,9 +277,28 @@ def load(self): self.paths.insert(0, path) self.load_file(path) - # Convert config_paths and distro_paths to lists of Path objects + # Convert config_paths to lists of Path objects self["config_paths"] = utils.Platform.expand_paths(self["config_paths"]) - self["distro_paths"] = utils.Platform.expand_paths(self["distro_paths"]) + + # Convert distro_paths to DistroFinder instances + distro_paths = [] + + default_distro_finder = self.get("entry_points", {}).get( + "hab.distro.finder.default", "hab.distro_finders.distro_finder:DistroFinder" + ) + for distro_finder in self["distro_paths"]: + if isinstance(distro_finder, str): + # Handle simple folder paths by converting to the DistroFinder class + distro_finder = [default_distro_finder, distro_finder] + + inst = self.entry_point_init( + "hab.distro.finder", distro_finder[0], distro_finder[1:] + ) + # Ensure these items can access the site and its cache + inst.site = self + distro_paths.append(inst) + + self["distro_paths"] = distro_paths # Ensure any platform_path_maps are converted to pathlib objects. self.standardize_platform_path_maps() @@ -344,10 +432,3 @@ def config_paths(self, config_paths): "config_paths", config_paths, cache, "*.json" ): yield dirname, path - - def distro_paths(self, distro_paths): - cache = self.cache.distro_paths() - for dirname, path, _ in self.cache.iter_cache_paths( - "distro_paths", distro_paths, cache, "*/.hab.json" - ): - yield dirname, path diff --git a/hab/utils.py b/hab/utils.py index 8969d79..66f2404 100644 --- a/hab/utils.py +++ b/hab/utils.py @@ -6,6 +6,7 @@ import os import re import sys +import tempfile import textwrap import zlib from abc import ABC, abstractmethod @@ -542,6 +543,14 @@ def default_ext(cls): """Returns the default file extension used on this platform.""" return cls._default_ext + @classmethod + def default_download_cache(cls): + """Path where download files are cached. + + This is used as the default location for `Site.downloads["cache_root"]`. + """ + return Path(tempfile.gettempdir()) / "hab_downloads" + @classmethod def expand_paths(cls, paths): """Converts path strings separated by ``cls.pathsep()`` and lists into