3/4: Add hab install system and switch to using DistroFinder

- Add `hab install` command that resolves all distros required to resolve
multiple URI's and then download and install any missing distros.
- Distro resolving is now handled by `DistroFinder` or sub-class.
- Site now has a downloads dictionary that configures how `hab install`
can resolve remote distros for installing locally.
- `Resolver.distro_mode_override` with context can be used to change
`Resolver.distro`'s output to the `Site.downloads["distros"] used to find
remote distros to install.

hab/__init__.py

@@ -1,10 +1,10 @@
-__all__ = ["__version__", "NotSet", "Resolver", "Site"]
+__all__ = ["__version__", "DistroMode", "NotSet", "Resolver", "Site"]
 
 from .utils import NotSet
 
 # Note: Future imports depend on NotSet so it must be imported first
 # isort: split
 
-from .resolver import Resolver
+from .resolver import DistroMode, Resolver
 from .site import Site
 from .version import version as __version__

hab/cache.py

@@ -111,6 +111,7 @@ def generate_cache(self, resolver, site_file, version=1):
         the provided site file. Use this method any time changes are made that
         hab needs to be aware of. Caching is enabled by the existence of this file.
         """
+        from .distro_finders.distro_finder import DistroFinder
         from .site import Site
 
         # Indicate the version specification this habcache file conforms to.
@@ -125,6 +126,9 @@ def generate_cache(self, resolver, site_file, version=1):
             glob_str, cls = stats
             # Process each glob dir defined for this site
             for dirname in temp_site.get(key, []):
+                # Caching is only supported for direct file paths
+                if isinstance(dirname, DistroFinder):
+                    dirname = dirname.root
                 cfg_paths = output.setdefault(key, {}).setdefault(
                     platform_path_key(dirname).as_posix(), {}
                 )
@@ -180,7 +184,7 @@ def iter_cache_paths(cls, name, paths, cache, glob_str=None, include_path=True):
                 logger.debug(f"Using glob for {name} dir: {dirname}")
                 # Fallback to globing the file system
                 if glob_str:
-                    paths = sorted(glob.glob(str(dirname / glob_str)))
+                    paths = utils.glob_path(dirname / glob_str)
                 else:
                     paths = []
             if not include_path:

hab/cli.py

@@ -9,7 +9,7 @@
 from click.shell_completion import CompletionItem
 from colorama import Fore
 
-from . import Resolver, Site, __version__, utils
+from . import DistroMode, Resolver, Site, __version__, utils
 from .parsers.unfrozen_config import UnfrozenConfig
 
 logger = logging.getLogger(__name__)
@@ -603,7 +603,18 @@ def env(settings, uri, launch):
     "--type",
     "report_type",
     type=click.Choice(
-        ["nice", "site", "s", "uris", "u", "versions", "v", "forest", "f", "all-uris"]
+        # Note: Put short names on same line as full name
+        # fmt: off
+        [
+            "nice",
+            "site", "s",
+            "uris", "u",
+            "versions", "v",
+            "downloads",
+            "forest", "f",
+            "all-uris",
+        ]
+        # fmt: on
     ),
     default="nice",
     help="Type of report.",
@@ -644,7 +655,7 @@ def dump(settings, uri, env, env_config, report_type, flat, verbosity, format_ty
 
     resolver = settings.resolver
 
-    if report_type in ("uris", "versions", "forest"):
+    if report_type in ("uris", "versions", "downloads", "forest"):
         from .parsers.format_parser import FormatParser
 
         formatter = FormatParser(verbosity, color=True)
@@ -659,16 +670,22 @@ def dump(settings, uri, env, env_config, report_type, flat, verbosity, format_ty
                     resolver.configs, fmt=formatter.format
                 ):
                     click.echo(line)
-        if report_type in ("versions", "forest"):
+        if report_type in ("versions", "downloads", "forest"):
             click.echo(f'{Fore.YELLOW}{" Versions ".center(50, "-")}{Fore.RESET}')
 
-            for line in resolver.dump_forest(
-                resolver.distros,
-                attr="name",
-                fmt=formatter.format,
-                truncate=truncate,
-            ):
-                click.echo(line)
+            mode = (
+                DistroMode.Downloaded
+                if report_type == "downloads"
+                else DistroMode.Installed
+            )
+            with resolver.distro_mode_override(mode):
+                for line in resolver.dump_forest(
+                    resolver.distros,
+                    attr="name",
+                    fmt=formatter.format,
+                    truncate=truncate,
+                ):
+                    click.echo(line)
     elif report_type == "all-uris":
         # Combines all non-placeholder URI's into a single json document and display.
         # This can be used to compare changes to configs when editing them in bulk.
@@ -786,6 +803,59 @@ def cache(settings, path):
     click.echo(f"Cache took: {e - s}, cache file: {out}")
 
 
+@_cli.command()
+@click.option(
+    "-u",
+    "--uri",
+    "uris",
+    multiple=True,
+    help="A URI that is resolved and all required distros are installed. Can "
+    "be used multiple times and each URI's distros are resolved independently.",
+)
+@click.option(
+    "-d",
+    "--distro",
+    "distros",
+    multiple=True,
+    help="Additional distros to install. Can be used multiple times and each use "
+    "is resolved independently.",
+)
+@click.option(
+    "--dry-run/--no-dry-run",
+    default=False,
+    help="Don't actually install anything, just print what would be installed.",
+)
+@click.option(
+    "--force-reinstall/--no-force-reinstall",
+    default=False,
+    help="Reinstall all resolved distros even if they are already installed.",
+)
+@click.option(
+    "--target",
+    type=click.Path(file_okay=False, resolve_path=True),
+    help="Install distros into DIRECTORY. Defaults to the sites "
+    'downloads["install_root"] setting.',
+)
+@click.pass_obj
+def install(settings, uris, distros, dry_run, force_reinstall, target):
+    """Install distros for use in hab. At least one uri or distro must be
+    specified to install. This is intended to install all versions of hab distros
+    that are required for a collection of hab URI on this system. This means that
+    unlike pip this may install multiple versions of hab distros.
+    """
+    distros = list(distros) if distros else None
+    uris = list(uris) if uris else None
+    if not distros and not uris:
+        raise ValueError("You must specify at least one --uri or --distro to install.")
+    settings.resolver.install(
+        uris=uris,
+        additional_distros=distros,
+        target=target,
+        dry_run=dry_run,
+        replace=force_reinstall,
+    )
+
+
 def cli(*args, **kwargs):
     """Runs the hab cli. If an exception is raised, only the exception message
     is printed and the stack trace is hidden. Use `hab -v ...` to enable showing