diff --git a/.github/workflows/python-static-analysis-and-test.yml b/.github/workflows/python-static-analysis-and-test.yml index 512681c..b8ee105 100644 --- a/.github/workflows/python-static-analysis-and-test.yml +++ b/.github/workflows/python-static-analysis-and-test.yml @@ -52,15 +52,6 @@ jobs: json_ver: ['json', 'json5'] os: ['ubuntu-latest', 'windows-latest'] python: ['3.7', '3.8', '3.9', '3.10', '3.11'] - # Works around the depreciation of python 3.6 for ubuntu - # https://github.com/actions/setup-python/issues/544 - include: - - json_ver: 'json' - os: 'ubuntu-20.04' - python: '3.6' - - json_ver: 'json5' - os: 'ubuntu-20.04' - python: '3.6' runs-on: ${{ matrix.os }} diff --git a/README.md b/README.md index a5c237c..2ba91ac 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ home directory on other platforms. ## Installing -Hab is installed using pip. It requires python 3.6 or above. It's recommended +Hab is installed using pip. It requires python 3.7 or above. It's recommended that you add the path to your python's bin or Scripts folder to the `PATH` environment variable so you can simply run the `hab` command. @@ -1367,7 +1367,7 @@ most part you can control the output using the `hab -v ...` verbosity option. However if you need more fine grained control you can create a `.hab_logging_prefs.json` file next to your user [user prefs](#user-prefs) file. The cli also supports passing the path to a configuration file using `hab --logging-config [path/to/file.json]` -that is used instead of the default file if pased. +that is used instead of the default file if passed. # Caveats @@ -1451,6 +1451,9 @@ hab in batch mode. Approximate time generated using `time cmd.exe /c "hab -h"` in git bash after omitting the `%py_exe% -m ...` call. +You can also set the `%TMP%` environment variable to a unique folder, but this +is more of a fix for scripted or unittest workflows. + # Glosary * **activate:** Update the current process(shell) for a given configuration. Name taken diff --git a/hab/__init__.py b/hab/__init__.py index fa27b9d..2fac716 100644 --- a/hab/__init__.py +++ b/hab/__init__.py @@ -1,10 +1,10 @@ -__all__ = ["__version__", "NotSet", "Resolver", "Site"] +__all__ = ["__version__", "DistroMode", "NotSet", "Resolver", "Site"] from .utils import NotSet # Note: Future imports depend on NotSet so it must be imported first # isort: split -from .resolver import Resolver +from .resolver import DistroMode, Resolver from .site import Site from .version import version as __version__ diff --git a/hab/cache.py b/hab/cache.py index 29559b9..51cc003 100644 --- a/hab/cache.py +++ b/hab/cache.py @@ -111,6 +111,7 @@ def generate_cache(self, resolver, site_file, version=1): the provided site file. Use this method any time changes are made that hab needs to be aware of. Caching is enabled by the existence of this file. """ + from .distro_finders.distro_finder import DistroFinder from .site import Site # Indicate the version specification this habcache file conforms to. @@ -125,6 +126,9 @@ def generate_cache(self, resolver, site_file, version=1): glob_str, cls = stats # Process each glob dir defined for this site for dirname in temp_site.get(key, []): + # Caching is only supported for direct file paths + if isinstance(dirname, DistroFinder): + dirname = dirname.root cfg_paths = output.setdefault(key, {}).setdefault( platform_path_key(dirname).as_posix(), {} ) @@ -152,9 +156,23 @@ def iter_cache_paths(cls, name, paths, cache, glob_str=None, include_path=True): """Yields path information stored in the cache falling back to glob if not cached. + Args: + name (str): The name of the cache being iterated. Often "config_paths" + or "distro_paths". + paths (list): A list of `pathlib.Path` paths to process. If this includes + glob paths they will be processed. + cache (dict): The cached data used if possible for each path. If a + path isn't in the cache, then will glob the path. + glob_str (str, optional): Added to each path if passed and a glob + is required. Ignored if the path is cached. + include_path (bool, optional): Controls how many items are yielded. + If True then each cached or globed path is yielded. Otherwise only + each path(dirname) is yielded and path is always None. + Yields: - dirname: Each path stored in paths. - path + dirname: Each path passed by paths. + path: The path to a given resource for this dirname. + cached: If the path was stored in a cache or required using glob. """ for dirname in paths: dn_posix = dirname.as_posix() @@ -166,7 +184,7 @@ def iter_cache_paths(cls, name, paths, cache, glob_str=None, include_path=True): logger.debug(f"Using glob for {name} dir: {dirname}") # Fallback to globing the file system if glob_str: - paths = sorted(glob.glob(str(dirname / glob_str))) + paths = utils.glob_path(dirname / glob_str) else: paths = [] if not include_path: diff --git a/hab/cli.py b/hab/cli.py index 579ae43..63160dd 100644 --- a/hab/cli.py +++ b/hab/cli.py @@ -9,7 +9,7 @@ from click.shell_completion import CompletionItem from colorama import Fore -from . import Resolver, Site, __version__, utils +from . import DistroMode, Resolver, Site, __version__, utils from .parsers.unfrozen_config import UnfrozenConfig logger = logging.getLogger(__name__) @@ -603,7 +603,18 @@ def env(settings, uri, launch): "--type", "report_type", type=click.Choice( - ["nice", "site", "s", "uris", "u", "versions", "v", "forest", "f", "all-uris"] + # Note: Put short names on same line as full name + # fmt: off + [ + "nice", + "site", "s", + "uris", "u", + "versions", "v", + "downloads", + "forest", "f", + "all-uris", + ] + # fmt: on ), default="nice", help="Type of report.", @@ -644,7 +655,7 @@ def dump(settings, uri, env, env_config, report_type, flat, verbosity, format_ty resolver = settings.resolver - if report_type in ("uris", "versions", "forest"): + if report_type in ("uris", "versions", "downloads", "forest"): from .parsers.format_parser import FormatParser formatter = FormatParser(verbosity, color=True) @@ -659,16 +670,22 @@ def dump(settings, uri, env, env_config, report_type, flat, verbosity, format_ty resolver.configs, fmt=formatter.format ): click.echo(line) - if report_type in ("versions", "forest"): + if report_type in ("versions", "downloads", "forest"): click.echo(f'{Fore.YELLOW}{" Versions ".center(50, "-")}{Fore.RESET}') - for line in resolver.dump_forest( - resolver.distros, - attr="name", - fmt=formatter.format, - truncate=truncate, - ): - click.echo(line) + mode = ( + DistroMode.Downloaded + if report_type == "downloads" + else DistroMode.Installed + ) + with resolver.distro_mode_override(mode): + for line in resolver.dump_forest( + resolver.distros, + attr="name", + fmt=formatter.format, + truncate=truncate, + ): + click.echo(line) elif report_type == "all-uris": # Combines all non-placeholder URI's into a single json document and display. # This can be used to compare changes to configs when editing them in bulk. @@ -786,6 +803,59 @@ def cache(settings, path): click.echo(f"Cache took: {e - s}, cache file: {out}") +@_cli.command() +@click.option( + "-u", + "--uri", + "uris", + multiple=True, + help="A URI that is resolved and all required distros are installed. Can " + "be used multiple times and each URI's distros are resolved independently.", +) +@click.option( + "-d", + "--distro", + "distros", + multiple=True, + help="Additional distros to install. Can be used multiple times and each use " + "is resolved independently.", +) +@click.option( + "--dry-run/--no-dry-run", + default=False, + help="Don't actually install anything, just print what would be installed.", +) +@click.option( + "--force-reinstall/--no-force-reinstall", + default=False, + help="Reinstall all resolved distros even if they are already installed.", +) +@click.option( + "--target", + type=click.Path(file_okay=False, resolve_path=True), + help="Install distros into DIRECTORY. Defaults to the sites " + 'downloads["install_root"] setting.', +) +@click.pass_obj +def install(settings, uris, distros, dry_run, force_reinstall, target): + """Install distros for use in hab. At least one uri or distro must be + specified to install. This is intended to install all versions of hab distros + that are required for a collection of hab URI on this system. This means that + unlike pip this may install multiple versions of hab distros. + """ + distros = list(distros) if distros else None + uris = list(uris) if uris else None + if not distros and not uris: + raise ValueError("You must specify at least one --uri or --distro to install.") + settings.resolver.install( + uris=uris, + additional_distros=distros, + target=target, + dry_run=dry_run, + replace=force_reinstall, + ) + + def cli(*args, **kwargs): """Runs the hab cli. If an exception is raised, only the exception message is printed and the stack trace is hidden. Use `hab -v ...` to enable showing diff --git a/hab/distro_finders/__init__.py b/hab/distro_finders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hab/distro_finders/cloud_zip.py b/hab/distro_finders/cloud_zip.py new file mode 100644 index 0000000..d38594e --- /dev/null +++ b/hab/distro_finders/cloud_zip.py @@ -0,0 +1,133 @@ +import logging +import pathlib +import time +import zipfile +from abc import ABCMeta, abstractmethod + +import remotezip +from cloudpathlib import CloudPath + +from .df_zip import DistroFinderZip + +logger = logging.getLogger(__name__) + + +class HabRemoteZip(remotezip.RemoteZip): + """`remotezip.RemoteZip` that doesn't call `close()` when exiting a with context. + + Opening a new RemoteZip instance is slow and changes depending on the size + of the .zip file. Cloud based workflow doesn't need to close the file pointer + like you need to when working on a local file. + """ + + def __exit__(self, type, value, traceback): + pass + + +class DistroFinderCloudZip(DistroFinderZip, metaclass=ABCMeta): + """Works with zipped distros stored remotely in Amazon S3 buckets. + + Working with zipped distros extracting the `hab_filename` information from + inside the .zip file. This is useful when you have direct access to the .zip + file. + + For `path`, this class uses a .zip `member path`. A member path is the absolute + path to the .zip joined with the member path of files contained inside the .zip + file. So if the archive file path is `c:/temp/dist_a_v0.1.zip` and the member is + `hab_filename`, then the member_path would be `c:/temp/dist_a_v0.1.zip/.hab.json`. + + Note: + This class should only be used to install distros in the hab download system. + + This expects one file to exist with a specific naming convention: + - `{distro}_v{version}.zip` contains the entire contents of the distro. + This should also contain the top level file `hab_filename`. When the distro + is installed and using hab normally this file will be used. The `hab_filename` + file's contents are extracted from the zip file and used to initialize the + `DistroVersion` returned by `self.distro` without being written to disk. + """ + + def __init__(self, root, site=None, safe=False, client=None): + # Only define client if it was passed, otherwise create it lazily. + if client: + self.client = client + super().__init__(root, site=site, safe=safe) + self._archives = {} + + def as_posix(self): + """Returns the root path as a posix style string.""" + if isinstance(self.root, CloudPath): + # CloudPath doesn't need as_posix + return str(self.root) + return super().as_posix() + + def cast_path(self, path): + """Return path cast to the `pathlib.Path` like class preferred by this class.""" + return CloudPath(path, client=self.client) + + @property + @abstractmethod + def client(self): + """A `cloudpathlib.client.Client` used to create `CloudPath` instances.""" + + @client.setter + @abstractmethod + def client(self, client): + pass + + @abstractmethod + def credentials(self): + """Returns the credentials needed for requests to connect to the cloud resource. + + Generates these credentials using the client object. + """ + + def archive(self, zip_path, partial=True): + """Returns a `zipfile.Zipfile` like instance for zip_path. + + Args: + zip_path (cloudpathlib.CloudPath): The path to the zip file to open. + partial (bool, optional): If True then you only need access to a small + part of the archive. If True then `HabRemoteZip` will be used + to only download specific files from the remote archive without + caching them to disk. If False then remote archives will be fully + downloaded to disk(using caching) before returning the open archive. + """ + if not partial or isinstance(zip_path, pathlib.PurePath): + logger.debug(f"Using CloudPath to open(downloading if needed) {zip_path}.") + archive = zipfile.ZipFile(zip_path) + archive.filename = zip_path + return archive + + # Creating a RemoteZip instance is very slow compared to local file access. + # Reuse existing objects if already created. + if zip_path in self._archives: + logger.debug(f"Reusing cloud .zip resource: {zip_path}") + return self._archives[zip_path] + + logger.debug(f"Connecting to cloud .zip resource: {zip_path}") + s = time.time() + auth, headers = self.credentials() + + archive = HabRemoteZip(zip_path.as_url(), auth=auth, headers=headers) + archive.filename = zip_path + e = time.time() + logger.info(f"Connected to cloud .zip resource: {zip_path}, took: {e - s}") + self._archives[zip_path] = archive + return archive + + def clear_cache(self, persistent=False): + """Clear cached data in memory. If `persistent` is True then also remove + cache data from disk if it exists. + """ + if persistent: + self.remove_download_cache() + super().clear_cache(persistent=persistent) + + # Ensure all cached archives are closed before clearing the cache. + for archive in self._archives.values(): + archive.close() + self._archives = {} + if persistent: + # Clear downloaded temp files + self.client.clear_cache() diff --git a/hab/distro_finders/df_zip.py b/hab/distro_finders/df_zip.py new file mode 100644 index 0000000..0deefd8 --- /dev/null +++ b/hab/distro_finders/df_zip.py @@ -0,0 +1,165 @@ +import logging + +from .. import utils +from .zip_sidecar import DistroFinderZipSidecar + +logger = logging.getLogger(__name__) + + +class DistroFinderZip(DistroFinderZipSidecar): + """Working with zipped distros extracting the `hab_filename` information from + inside the .zip file. This is useful when you have direct access to the .zip + file. + + For `path`, this class uses a .zip `member path`. A member path is the absolute + path to the .zip joined with the member path of files contained inside the .zip + file. So if the archive file path is `c:/temp/dist_a_v0.1.zip` and the member is + `hab_filename`, then the member_path would be `c:/temp/dist_a_v0.1.zip/.hab.json`. + + Note: + This class should only be used to install distros in the hab download system. + + This expects one file to exist with a specific naming convention: + - `{distro}_v{version}.zip` contains the entire contents of the distro. + This should also contain the top level file `hab_filename`. When the distro + is installed and using hab normally this file will be used. The `hab_filename` + file's contents are extracted from the zip file and used to initialize the + `DistroVersion` returned by `self.distro` without being written to disk. + """ + + def __init__(self, root, site=None, safe=True): + super().__init__(root, site=site) + self.glob_str = "*.zip" + self._cache = {} + self.safe = safe + + def clear_cache(self, persistent=False): + """Clear cached data in memory. If `persistent` is True then also remove + cache data from disk if it exists. + """ + self._cache = {} + + def content(self, path): + """Returns the distro container for a given path as `pathlib.Path`. + + For this class it returns the path to the .zip file. This .zip file + contains the contents of the distro and the actual `hab_filename` used + to create the distro. + + Args: + path (pathlib.Path): The member path to the `hab_filename` file defining + the distro. + """ + # If path is already a .zip file return it. + # Note: We can't concatenate this with `pathlib.Path.parents` so this has + # to be done separately from the for loop later + if path.suffix.lower() == ".zip": + return path + + # Search for the right most .zip file extension and return that path if found + for parent in path.parents: + if parent.suffix.lower() == ".zip": + return parent + + # Otherwise fall back to returning the path + return path + + def content_member(self, path): + """Splits a member path into content and member. + + Args: + path (os.PathLike): The member path to split. + + Returns: + content(os.PathLike): Path to the .zip file. + member (str): Any remaining member path after the .zip file. If path + doesn't specify a member, then a empty string is returned. + """ + content = self.content(path) + member = str(path.relative_to(content)) + # Return a empty string instead of the relative dot + if member == ".": + member = "" + return content, member + + def distro_path_info(self): + """Generator yielding distro info for each distro found by this distro finder. + + Note: + This class doesn't use habcache features so cached will always be `False`. + + Yields: + dirname: Will always be `None`. This class deals with only compressed + .zip files so there is not a parent directory to work with. + path: The member path to a given resource. + cached: Will always be `False`. The path is not stored in a .habcache + file so this data is not cached across processes. + """ + for path in self.root.glob(self.glob_str): + member_path = path / self.hab_filename + if self.safe: + # Opening archives on cloud based systems is slow, this allows us + # to disable checking that the archive actually has a `hab_filename` file. + data = self.get_file_data(member_path) + # This should only return None if the archive doesn't contain member + if data is None: + continue + + yield None, member_path, False + + def get_file_data(self, path): + """Return the data stored inside a member of a .zip file as bytes. + + This is cached and will only open the .zip file to read the contents the + first time path is used for this instance. + + Args: + path: The member path to a given resource. If the path points directly + to a .zip file then member is assumed to be `self.hab_filename`. + """ + content, member = self.content_member(path) + if not member: + member = self.hab_filename + path = path / member + logger.debug(f'Implicitly added member "{member}" to path "{path}".') + + if path in self._cache: + return self._cache[path] + + with self.archive(content) as archive: + if member in archive.namelist(): + data = archive.read(member) + else: + data = None + self._cache[path] = data + + return self._cache[path] + + def load_path(self, path): + """Returns a raw dictionary use to create a `DistroVersion` with version set. + + Returns the actual contents of the .zip file's top level file `hab_filename` + without writing that data to disk. The return is passed to `DistroVersion.load` + as the data argument. This allows the `DistroFinder` class to directly use + the data contained inside the .zip archive. + + The version property will always be set in the return. If not defined + in the `hab_filename` file's contents, its set to the return of `version_for_path`. + + Args: + path (pathlib.Path): The member path to the `hab_filename` file inside + of the .zip file. + + Raises: + KeyError: This method uses the cache populated by `distro_path_info` + and that method needs to be called before calling this. It is also + raised if the requested `path` is not defined in the distro. + """ + logger.debug(f'Loading json: "{path}"') + data = self.get_file_data(path) + data = data.decode("utf-8") + data = utils.loads_json(data, source=path) + # Pull the version from the sidecar filename if its not explicitly set + if "version" not in data: + _, data["version"] = self.version_for_path(path) + return data diff --git a/hab/distro_finders/distro_finder.py b/hab/distro_finders/distro_finder.py new file mode 100644 index 0000000..4b549ba --- /dev/null +++ b/hab/distro_finders/distro_finder.py @@ -0,0 +1,160 @@ +import logging +import pathlib +import shutil + +from colorama import Fore, Style + +from .. import utils +from ..errors import InstallDestinationExistsError +from ..parsers.distro_version import DistroVersion + +logger = logging.getLogger(__name__) + + +class DistroFinder: + """A class used to find and install distros if required. + + The class `DistroFinder` is used by hab in normal operation. Most of the the + other sub-classes like `DistroFinderZip` are used by the hab download system + to download and extract distro versions into the expected folder structure. + + The aliases(programs) hab launches are normally not designed to load files + from inside of a .zip file so the contents of the distro need to be expanded + into a directory structure the alias can process. + """ + + hab_filename = ".hab.json" + """The file hab uses to define the distro relative to its root.""" + + def __init__(self, root, site=None): + self.site = site + self.root = utils.Platform.normalize_path(self.cast_path(root)) + self.glob_str = f"*/{self.hab_filename}" + + def __eq__(self, other): + if not hasattr(other, "root"): + return False + if not hasattr(other, "glob_str"): + return False + return self.root == other.root and self.glob_str == other.glob_str + + def __str__(self): + return f"{self.root}" + + def as_posix(self): + """Returns the root path as a posix style string.""" + return self.root.as_posix() + + def cast_path(self, path): + """Return path cast to the `pathlib.Path` like class preferred by this class.""" + return pathlib.Path(path) + + def clear_cache(self, persistent=False): + """Clear cached data in memory. If `persistent` is True then also remove + cache data from disk if it exists. + """ + pass + + def content(self, path): + """Returns the distro container for a given path as `pathlib.Path`. + + The default implementation returns the directory containing the `hab_filename` + file but subclasses may return other objects like .zip files. + + Args: + path (pathlib.Path): The path to the `hab_filename` file defining the distro. + """ + return path.parent + + def distro(self, forest, resolver, path): + """Returns an `DistroVersion` instance for the distro described py path. + + Args: + forest: A dictionary of hab.parser objects used to initialize the return. + resolver (hab.Resolver): The Resolver used to initialize the return. + path (pathlib.Path): The path to the `hab_filename` file defining the + distro. This path is loaded into the returned instance. + """ + distro = DistroVersion(forest, resolver, path, root_paths=set((self.root,))) + distro.finder = self + return distro + + def distro_path_info(self): + """Generator yielding distro info for each distro found by this distro finder. + + Note: + To use habcache features you must set the site property of this class + to the desired `hab.site.Site` class. If you don't then it will always + glob its results and cached will always be False. + + Yields: + dirname: Each path passed by paths. + path: The path to a given resource for this dirname. + cached: If the path was stored in a .habcache file or required using glob. + """ + # Handle if site has not been set, this does not use habcache. + if not self.site: + logger.debug("site not set, using direct glob.") + for path in utils.glob_path(self.root / self.glob_str): + yield self.root, self.cast_path(path), False + + # Otherwise use the site cache to yield the results + cache = self.site.cache.distro_paths() + for dirname, path, cached in self.site.cache.iter_cache_paths( + "distro_paths", [self.root], cache, self.glob_str + ): + yield dirname, path, cached + + def dump(self, verbosity=0, color=None, width=80): + """Return string representation of this object with various verbosity.""" + if verbosity > 1: + if not color: + return f"{self.root} [{type(self).__name__}]" + return f"{self.root} {Fore.CYAN}[{type(self).__name__}]{Style.RESET_ALL}" + return str(self) + + def install(self, path, dest): + """Install the distro into dest. + + Args: + path (pathlib.Path): The path to the `hab_filename` file defining the + distro. This path is used to find the `content` of the distro. + dest (pathlib.Path or str): The directory to install the distro into. + The contents of the distro are installed into this directory. + All intermediate directories needed to contain dest will be created. + """ + path = self.content(path) + if dest.exists(): + raise InstallDestinationExistsError(dest) + logger.debug(f"Installing to {dest} from source {path}") + shutil.copytree(path, dest) + + def installed(self, path): + """Returns if the hab_filename exists inside the root location""" + return (path / self.hab_filename).exists() + + def load_path(self, path): + """Returns a raw dictionary use to create a `DistroVersion` or None. + + The return is passed to `DistroVersion.load` as the data argument. This + allows the `DistroFinder` class to bypass the normal json loading method + for distros. + + This is called by `distro` and used by sub-classes to more efficiently + load the distro dictionary when possible. The default class returns `None`. + + Args: + path (pathlib.Path): The path to the `hab_filename` file defining the + distro used to define the returned data. + """ + # By default return None to use the `DistroVersion._load` method. + return None + + @property + def site(self): + """A `hab.site.Site` instance used to enable habcache.""" + return self._site + + @site.setter + def site(self, site): + self._site = site diff --git a/hab/distro_finders/s3_zip.py b/hab/distro_finders/s3_zip.py new file mode 100644 index 0000000..1c820fd --- /dev/null +++ b/hab/distro_finders/s3_zip.py @@ -0,0 +1,84 @@ +import logging +from hashlib import sha256 + +from cloudpathlib import S3Client +from requests_aws4auth import AWS4Auth + +from .. import utils +from .cloud_zip import DistroFinderCloudZip + +logger = logging.getLogger(__name__) + + +class DistroFinderS3Zip(DistroFinderCloudZip): + """Works with zipped distros stored remotely in Amazon S3 buckets. + + Working with zipped distros extracting the `hab_filename` information from + inside the .zip file. This is useful when you have direct access to the .zip + file. + + For `path`, this class uses a .zip `member path`. A member path is the absolute + path to the .zip joined with the member path of files contained inside the .zip + file. So if the archive file path is `c:/temp/dist_a_v0.1.zip` and the member is + `hab_filename`, then the member_path would be `c:/temp/dist_a_v0.1.zip/.hab.json`. + + Note: + This class should only be used to install distros in the hab download system. + + This expects one file to exist with a specific naming convention: + - `{distro}_v{version}.zip` contains the entire contents of the distro. + This should also contain the top level file `hab_filename`. When the distro + is installed and using hab normally this file will be used. The `hab_filename` + file's contents are extracted from the zip file and used to initialize the + `DistroVersion` returned by `self.distro` without being written to disk. + """ + + def __init__(self, root, site=None, safe=False, client=None, profile_name=None): + self.profile_name = profile_name + super().__init__(root, site=site, safe=safe, client=client) + + @property + def client(self): + try: + return self._client + except AttributeError: + kwargs = {} + if self.profile_name: + kwargs["profile_name"] = self.profile_name + if self.site: + kwargs["local_cache_dir"] = self.site.downloads["cache_root"] + else: + kwargs["local_cache_dir"] = utils.Platform.default_download_cache() + self._client = S3Client(**kwargs) + return self._client + + @client.setter + def client(self, client): + self._client = client + + def credentials(self): + """Returns the credentials needed for requests to connect to aws s3 bucket. + + Generates these credentials using the client object. + """ + + try: + return self._credentials + except AttributeError: + pass + # The `x-amz-content-sha256` header is required for all AWS Signature + # Version 4 requests. It provides a hash of the request payload. If + # there is no payload, you must provide the hash of an empty string. + headers = {"x-amz-content-sha256": sha256(b"").hexdigest()} + + location = self.client.client.get_bucket_location(Bucket=self.root.bucket)[ + "LocationConstraint" + ] + auth = AWS4Auth( + refreshable_credentials=self.client.sess.get_credentials(), + region=location, + service="s3", + ) + + self._credentials = (auth, headers) + return self._credentials diff --git a/hab/distro_finders/zip_sidecar.py b/hab/distro_finders/zip_sidecar.py new file mode 100644 index 0000000..3fee689 --- /dev/null +++ b/hab/distro_finders/zip_sidecar.py @@ -0,0 +1,154 @@ +import logging +import re +import zipfile + +from packaging.version import VERSION_PATTERN + +from .. import utils +from ..errors import InstallDestinationExistsError +from ..parsers.lazy_distro_version import LazyDistroVersion +from .distro_finder import DistroFinder + +logger = logging.getLogger(__name__) + + +class DistroFinderZipSidecar(DistroFinder): + """Working with zipped distros that have a sidecar `dist_name_v0.0.0.hab.json` + file. This is useful when it can't extract the `hab_filename` from the .zip file. + + Note: + This class should only be used to install distros in the hab download system. + + This expects two files to exist with a specific naming convention: + - `{distro}_v{version}.zip` contains the entire contents of the distro. + This should also contain the top level file `hab_filename`. When the distro + is installed and using hab normally this file will be used. + - `{distro}_v{version}.hab.json` is a copy of the .hab.json file contained + in the .zip file. This file is used to initialize the `DistroVersion` + returned by `self.distro` and mainly used to resolve dependent distros + efficiently when running hab download. It is not used outside of this class. + """ + + version_regex = re.compile( + rf"(?P[^\\/]+)_v{VERSION_PATTERN}", flags=re.VERBOSE | re.IGNORECASE + ) + """Regex used to parse the distro name and version from a file path. This looks + for a string matching {distro}_v{version}. + """ + + def __init__(self, root, site=None): + super().__init__(root, site) + self.glob_str = "*.hab.json" + + def archive(self, zip_path, partial=True): + """Returns a `zipfile.Zipfile` like instance for zip_path. + + Args: + zip_path (os.PathLike): The path to the zip file to open. + partial (bool, optional): If True then you only need access to a small + part of the archive. This is used by sub-classes to optimize access + to remote zip archives. If True then `HabRemoteZip` will be used + to only download specific files from the remote archive without + caching them to disk. If False then remote archives will be fully + downloaded to disk(using caching) before returning the open archive. + """ + return zipfile.ZipFile(zip_path) + + def content(self, path): + """Returns the distro container for a given path as `pathlib.Path`. + + For this class it returns the path to the sidecar .zip file. This .zip + file contains the contents of the distro. + + Args: + path (pathlib.Path): The path to the `hab_filename` file defining the distro. + """ + # This simply replaces `hab_filename` with `.zip`. + return path.with_suffix("").with_suffix(".zip") + + def content_member(self, path): + """Splits a member path into content and member. + + Args: + path (os.PathLike): The member path to split. + + Returns: + content(os.PathLike): Path to the .zip file. + member (str): This class always returns `hab_filename`. + """ + content = self.content(path) + return content, self.hab_filename + + def distro(self, forest, resolver, path): + """Returns an `DistroVersion` instance for the distro described py path. + + Args: + forest: A dictionary of hab.parser objects used to initialize the return. + resolver (hab.Resolver): The Resolver used to initialize the return. + path (pathlib.Path): The path to the `hab_filename` file defining the + distro. This path is loaded into the returned instance. + """ + distro = LazyDistroVersion(forest, resolver, root_paths=set((self.root,))) + distro.finder = self + distro.name, distro.version = self.version_for_path(path) + distro.distro_name = distro.name + distro.load(path) + return distro + + def install(self, path, dest, replace=False): + """Install the distro into dest. + + Args: + path (os.PathLike): The path to the `hab_filename` file defining the + distro. This path is used to find the `content` of the distro. + dest (pathlib.Path): The directory to install the distro into. + The contents of the distro are installed into this directory. + All intermediate directories needed to contain dest will be created. + replace (bool, optional): If the distro already exists, remove and + re-copy it. Otherwise raises an Exception. + """ + path, member = self.content_member(path) + if (dest / member).exists(): + if not replace: + raise InstallDestinationExistsError(dest) + + logger.debug(f"Extracting to {dest} from zip {path}") + with self.archive(path, partial=False) as archive: + members = archive.namelist() + total = len(members) + for i, member in enumerate(members): + logger.debug(f"Extracting file({i}/{total}): {member}") + archive.extract(member, dest) + return True + + def load_path(self, path): + """Returns a raw dictionary use to create a `DistroVersion` with version set. + + The return is passed to `DistroVersion.load` as the data argument. This + allows the `DistroFinder` class to bypass the normal json loading method + for distros. + + Returns the contents of the sidecar `{distro}_v{version}.hab.json` file. + The version property will always be set in the return. If not defined + in the file's contents, its set to the return of `version_for_path`. + + Args: + path (pathlib.Path): The path to the `hab_filename` file defining the + distro used to define the returned data. + """ + logger.debug(f'Loading "{path}"') + data = utils.load_json_file(path) + # Pull the version from the sidecar filename if its not explicitly set + if "version" not in data: + _, data["version"] = self.version_for_path(path) + return data + + def version_for_path(self, path): + """Returns the distro name and version for the given path as a string. + + Args: + path (pathlib.Path): The path to the `*.hab.json` file defining the + distro. Uses the `version_regex` to parse the version release. + """ + result = self.version_regex.search(str(path)) + return result.group("name"), result.group("release") diff --git a/hab/errors.py b/hab/errors.py index f3cc0dc..cba4214 100644 --- a/hab/errors.py +++ b/hab/errors.py @@ -1,3 +1,6 @@ +import errno + + class HabError(Exception): """Base class for all hab errors.""" @@ -72,6 +75,13 @@ def __str__(self): return ret +class InstallDestinationExistsError(HabError, FileExistsError): + """Raised if attempting to install to a directory that already exists.""" + + def __init__(self, filename, message="The destination already exists"): + super().__init__(errno.EEXIST, message, filename) + + class ReservedVariableNameError(HabError): """Raised if a custom variable uses a reserved variable name.""" diff --git a/hab/merge_dict.py b/hab/merge_dict.py index 561bf69..7873bb4 100644 --- a/hab/merge_dict.py +++ b/hab/merge_dict.py @@ -53,18 +53,23 @@ def default_format(self, value, platform=None): """Apply string formatting rules to the given value if applicable. If value is a list, this method is recursively called on the contents - and a new list is returned. If a bool or dict are passed, they are - returned without modification. Otherwise its assume to be a string and - str.format is called passing `self.format_kwargs`. + and a new list is returned. It is similarly called on the values of dict's. + If a bool or int are passed, they are returned without modification. + Otherwise its assume to be a string and str.format is called passing + `self.format_kwargs`. If `self.site` is set and platform is passed, site.platform_path_map is called on the text output to convert it to the desired platform. """ + if value is None: + return value if isinstance(value, list): # Format the individual items if a list of args is used. # return [v.format(**self.format_kwargs) for v in value] return [self.default_format(v) for v in value] - if isinstance(value, (bool, dict, int)): + if isinstance(value, dict): + return {k: self.formatter(v, platform=platform) for k, v in value.items()} + if isinstance(value, (bool, int)): return value ret = value.format(**self.format_kwargs) diff --git a/hab/parsers/distro_version.py b/hab/parsers/distro_version.py index ed0f488..dfab343 100644 --- a/hab/parsers/distro_version.py +++ b/hab/parsers/distro_version.py @@ -16,6 +16,7 @@ class DistroVersion(HabBase): def __init__(self, *args, **kwargs): self._alias_mods = NotSet + self._finder = None super().__init__(*args, **kwargs) def _cache(self): @@ -96,6 +97,15 @@ def alias_mods(self): """ return self._alias_mods + @hab_property(verbosity=3) + def finder(self): + """The DistroFinder instance used to create this instance.""" + return self._finder + + @finder.setter + def finder(self, value): + self._finder = value + def _load(self, filename, cached=True): """Sets self.filename and parses the json file returning the data.""" ret = super()._load(filename, cached=cached) @@ -108,9 +118,10 @@ def _load(self, filename, cached=True): ret["version"] = str(self.version) return ret - def load(self, filename): + def load(self, filename, data=None): # Fill in the DistroVersion specific settings before calling super - data = self._load(filename) + if data is None: + data = self._load(filename) # The name should be the version == specifier. self.distro_name = data.get("name") diff --git a/hab/parsers/hab_base.py b/hab/parsers/hab_base.py index e4eb7b0..a79d8b5 100644 --- a/hab/parsers/hab_base.py +++ b/hab/parsers/hab_base.py @@ -473,7 +473,9 @@ def filename(self, filename): self._filename = Path(os.devnull) self._dirname = Path(os.devnull) else: - self._filename = Path(filename) + if isinstance(filename, str): + filename = Path(filename) + self._filename = filename self._dirname = self._filename.parent def format_environment_value(self, value, ext=None, platform=None): @@ -594,12 +596,16 @@ def load(self, filename, data=None): """Load this objects configuration from the given json filename. Args: - filename (str): The json file to load the config from. + filename: The file to load the config from. If this is a string it is + cast to a `pathlib.Path` object, otherwise it is expected to be + a `pathlib.Path` like object. data (dict, optional): If provided this dict is used instead of parsing the json file. In this case filename is ignored. """ if data is None: data = self._load(filename) + else: + self.filename = filename # Check for NotSet so sub-classes can set values before calling super if self.name is NotSet: diff --git a/hab/parsers/lazy_distro_version.py b/hab/parsers/lazy_distro_version.py new file mode 100644 index 0000000..4c13e75 --- /dev/null +++ b/hab/parsers/lazy_distro_version.py @@ -0,0 +1,127 @@ +import logging +import shutil +from pathlib import Path + +from .. import NotSet +from ..errors import InstallDestinationExistsError +from .distro_version import DistroVersion + +logger = logging.getLogger(__name__) + + +class DistroPath: + __slots__ = ("distro", "hab_filename", "root", "site") + + def __init__(self, distro, root, relative=NotSet, site=None): + self.distro = distro + self.site = site + if isinstance(root, str): + root = Path(root) + + if relative is NotSet: + if site and "relative_path" in site.downloads: + relative = site.downloads["relative_path"] + else: + relative = "{distro_name}/{version}" + + if relative: + root = root / relative.format( + name=self.distro.name, + distro_name=self.distro.distro_name, + version=self.distro.version, + ) + + self.hab_filename = root / ".hab.json" + self.root = root + + +class LazyDistroVersion(DistroVersion): + """A DistroVersion class that loads data on first access. + + This class will raise a ValueError if filename is passed. This class expects + that after initializing you set the properties `name`, `version`, `finder`, + and `distro_name` After that you should call `load(filename)`. + + TODO: Add overrides to each getter/setter on this class like we have done to + the distros property. + """ + + def __init__(self, *args, **kwargs): + if len(args) > 2 or "filename" in kwargs: + raise ValueError("Passing filename to this class is not supported.") + + self._loaded = False + self._loaded_data = NotSet + super().__init__(*args, **kwargs) + + @DistroVersion.distros.getter + def distros(self): + """A list of all of the requested distros to resolve.""" + self._ensure_loaded() + return super().distros + + def install(self, dest, replace=False, relative=NotSet): + """Install the distro into dest. + + Installs the distro into `dest / relative` creating any intermediate + directories needed. In most cases you would pass one of your site's + `distro_paths` to `dist` and the default relative value creates the + recommended distro/version/contents folder structure that ensures that + each distro version doesn't conflict with any others. + + Args: + dest (pathlib.Path or str): The base directory to install this distro + into. This is joined with `relative` to create the full path. + replace (bool, optional): If dest already contains this distro, this + will remove the existing install then re-install the distro. + relative (str, optional): Additional path items joined to dest after + being formatted. The kwargs "name", "distro_name" and "version" + are passed to the `str.format` called on this variable. + + Raises: + hab.errors.InstallDestinationExistsError: If the requested dest already + contains this distro this error is raised and it is not installed. + Unless `replace` is set to True. + """ + if not isinstance(dest, DistroPath): + dest = DistroPath(self, dest, relative=relative, site=self.resolver.site) + + installed = self.installed(dest, relative=relative) + if installed: + if not replace: + raise InstallDestinationExistsError(dest.root) + # Replace requested, remove the existing files before continuing. + logger.info(f"Removing existing distro install: {dest.root}") + shutil.rmtree(dest.root) + + self.finder.install(self.filename, dest.root) + # The resolver cache is now out of date, force it to refresh on next access. + self.resolver.clear_caches() + + def installed(self, dest, relative=NotSet): + if not isinstance(dest, DistroPath): + dest = DistroPath(self, dest, relative=relative, site=self.resolver.site) + return dest.hab_filename.exists() + + def _ensure_loaded(self): + """Ensures the data is loaded. + + On first call this method actually processes the loading of data for + this DistroVersion. Any additional calls are ignored. + """ + if self._loaded: + return + + self._loaded = True + data = self.finder.load_path(self.filename) + return super().load(self.filename, data=data) + + def load(self, filename, data=NotSet): + # The name should be the version == specifier. + self.name = f"{self.distro_name}=={self.version}" + + self.filename = filename + self._loaded_data = data + self._loaded = False + self.context = [self.distro_name] + return data diff --git a/hab/resolver.py b/hab/resolver.py index bc7bf72..6a6f3e0 100644 --- a/hab/resolver.py +++ b/hab/resolver.py @@ -1,14 +1,17 @@ # __all__ = ["Resolver"] +import concurrent.futures import copy +import enum import logging +from contextlib import contextmanager import anytree from packaging.requirements import Requirement from . import utils -from .errors import _IgnoredVersionError -from .parsers import Config, DistroVersion, HabBase +from .errors import HabError, _IgnoredVersionError +from .parsers import Config, HabBase from .site import Site from .solvers import Solver from .user_prefs import UserPrefs @@ -16,6 +19,17 @@ logger = logging.getLogger(__name__) +class DistroMode(enum.Enum): + """Used by `hab.Revolver` to control which forest is used to resolve distros.""" + + # TODO: Switch docstrings to `Annotated` if we move to py 3.9+ only + # support https://stackoverflow.com/a/78361486 + Downloaded = enum.auto() + """Use the `downloadable_distros` forest when using `Resolver.distros`.""" + Installed = enum.auto() + """Use the `installed_distros` forest when using `Resolver.distros`.""" + + class Resolver(object): """Used to resolve a hab environment setup and apply it to the current environment. @@ -67,7 +81,9 @@ def __init__( logger.debug(f"distro_paths: {self.distro_paths}") self._configs = None - self._distros = None + self.distro_mode = DistroMode.Installed + self._downloadable_distros = None + self._installed_distros = None self.ignored = self.site["ignored_distros"] # If true, then all scripts are printed instead of being written to disk @@ -80,8 +96,10 @@ def clear_caches(self): """Clears cached resolved data so it is re-generated on next use.""" logger.debug("Resolver cache cleared.") self._configs = None - self._distros = None + self._downloadable_distros = None + self._installed_distros = None self.site.cache.clear() + [distro_finder.clear_cache() for distro_finder in self.distro_paths] def closest_config(self, path, default=False): """Returns the most specific leaf or the tree root matching path. Ignoring any @@ -149,6 +167,7 @@ def config_paths(self): @config_paths.setter def config_paths(self, paths): + """Path's used to populate `configs`.""" # Convert string paths into a list if isinstance(paths, str): paths = utils.Platform.expand_paths(paths) @@ -164,8 +183,33 @@ def configs(self): self._configs = self.parse_configs(self.config_paths) return self._configs + @contextmanager + def distro_mode_override(self, mode): + """A context manager that sets `distro_mode` while inside the context. + This lets you switch which distro forest is returned by the `distro` method. + + Example: + + assert resolver.distros == resolver.installed_distros + with resolver.distro_mode_override(DistroMode.Downloaded): + assert resolver.distros == resolver.downloadable_distros + assert resolver.distros == resolver.installed_distros + """ + if not isinstance(mode, DistroMode): + raise ValueError("You can only specify DistroModes.") + + current = self.distro_mode + logger.debug(f"Setting Resolver distro_mode to {mode} from {current}.") + try: + self.distro_mode = mode + yield current + finally: + self.distro_mode = current + logger.debug(f"Restored distro_mode to {self.distro_mode}.") + @property def distro_paths(self): + """`DistroFinder`s used to populate `installed_distros`.""" return self.site["distro_paths"] @distro_paths.setter @@ -175,15 +219,46 @@ def distro_paths(self, paths): paths = utils.Platform.expand_paths(paths) self.site["distro_paths"] = paths - # Reset _distros so we re-generate them the next time they are requested - self._distros = None + # Reset the cache so we re-generate it the next time it is requested. + self._installed_distros = None + + @property + def installed_distros(self): + """A dictionary of all usable distros that have been parsed for this resolver. + + These are the distros used by hab when a hab environment is configured + and aliases(programs) access these files. + """ + if self._installed_distros is None: + self._installed_distros = self.parse_distros(self.distro_paths) + return self._installed_distros @property def distros(self): - """A list of all of the requested distros to resolve.""" - if self._distros is None: - self._distros = self.parse_distros(self.distro_paths) - return self._distros + """A dictionary of distros for this resolver. + + This forest is used to resolve distro dependencies into config versions. + + The output is dependent on `self.distro_mode`, use the `distro_mode_override` + context manager to change the mode temporarily. + """ + if self.distro_mode == DistroMode.Downloaded: + return self.downloadable_distros + return self.installed_distros + + @property + def downloadable_distros(self): + """A dictionary of all distros that can be installed into `installed_distros`. + + This is used by the hab install process, not when enabling a hab + environment. These distros are available to download and install for use + in the `installed_distros` forest. + """ + if self._downloadable_distros is None: + self._downloadable_distros = self.parse_distros( + self.site.downloads["distros"] + ) + return self._downloadable_distros @classmethod def dump_forest( @@ -290,6 +365,98 @@ def freeze_configs(self): out[uri] = cfg.freeze() return out + def install( + self, + uris=None, + additional_distros=None, + target=None, + dry_run=True, + replace=False, + ): + """Ensure the required distros are installed for use in hab. + + Resolves the distros defined by one or more URI's and additional distros + against the distro versions available on from a `downloadable_distros`. + Then extracts them into a target location for use in hab environments. + + Each URI and additional_distros requirement is resolved independently. This + allows you to install multiple versions of a given distro so the correct + one is available when a given URI is used in a hab environment. + + Args: + uris (list, optional): A list of URI strings. These URI's are resolved + against the available distros in `downloadable_distros`. + additional_distros (list, optional): A list of additional distro + requirements to resolve and install. + target (os.PathLike, optional): The target directory to install all + resolved distros into. This is the root directory. The per-distro + name and version paths are added relative to this directory based + on the `site.downloads["relative_path"]` setting. + dry_run (bool, optional): If True then don't actually install the + the distros. The returned list is the final list of all distros + that need to be installed. + replace (bool, optional): This method skips installing any distros + that are already installed. Setting this to True will delete the + existing distros before re-installing them. + + Returns: + list: A list of DistroVersion's that were installed. The distros are + from `downloadable_distros` and represent the remote resources. + """ + + def str_distros(distros, sep=", "): + return sep.join(sorted([d.name for d in missing])) + + if target is None: + target = self.site.downloads.get("install_root") + if target is None: + raise HabError( + 'You must specify target, or set ["downloads"]["install_root"] ' + "in your site config." + ) + + distros = set() + if uris is None: + uris = [] + # Resolve all distros and any additional distros they may require from + # the download forest. + with self.distro_mode_override(DistroMode.Downloaded): + if additional_distros: + requirements = self.resolve_requirements(additional_distros) + for req in requirements.values(): + version = self.find_distro(req) + distros.add(version) + + for uri in uris: + cfg = self.resolve(uri) + distros.update(cfg.versions) + + # Check the installed forest for any that are not already installed and + # build a list of missing distros. + missing = [] + for distro in distros: + if replace: + missing.append(distro) + continue + + installed = self.find_distro(distro.distro_name) + if not installed: + missing.append(distro) + + if dry_run: + logger.warning(f"Dry Run would install distros: {str_distros(missing)}") + return missing + + # Download and install the missing distros using threading to speed up + # the download process. + logger.warning(f"Installing distros: {str_distros(missing)}") + with concurrent.futures.ThreadPoolExecutor() as executor: + for distro in missing: + logger.warning(f"Installing distro: {distro.name}") + executor.submit(distro.install, target, replace=replace) + logger.warning(f"Installed distros: {str_distros(missing)}") + return missing + @classmethod def instance(cls, name="main", **kwargs): """Returns a shared Resolver instance for name, initializing it if required. @@ -324,14 +491,16 @@ def parse_configs(self, config_paths, forest=None): Config(forest, self, path, root_paths=set((dirname,))) return forest - def parse_distros(self, distro_paths, forest=None): + def parse_distros(self, distro_finders, forest=None): + """Parse all provided DistroFinders and populate the forest of distros.""" if forest is None: forest = {} - for dirname, path in self.site.distro_paths(distro_paths): - try: - DistroVersion(forest, self, path, root_paths=set((dirname,))) - except _IgnoredVersionError as error: - logger.debug(str(error)) + for distro_finder in distro_finders: + for _, path, _ in distro_finder.distro_path_info(): + try: + distro_finder.distro(forest, self, path) + except _IgnoredVersionError as error: + logger.debug(str(error)) return forest def resolve(self, uri, forced_requirements=None): @@ -375,6 +544,9 @@ def resolve_requirements(self, requirements, omittable=None): requirements. """ + if isinstance(requirements, list): + requirements = Solver.simplify_requirements(requirements) + solver = Solver( requirements, self, forced=self.forced_requirements, omittable=omittable ) diff --git a/hab/site.py b/hab/site.py index 4cab722..8593ff8 100644 --- a/hab/site.py +++ b/hab/site.py @@ -4,6 +4,7 @@ from pathlib import Path, PurePosixPath, PureWindowsPath from colorama import Fore, Style +from importlib_metadata import EntryPoint from . import utils from .cache import Cache @@ -39,6 +40,7 @@ def __init__(self, paths=None, platform=None): if platform is None: platform = utils.Platform.name() self.platform = platform + self._downloads_parsed = False # Add default data to all site instances. Site data is only valid for # the current platform, so discard any other platform configurations. @@ -64,6 +66,50 @@ def __init__(self, paths=None, platform=None): def data(self): return self.frozen_data.get(self.platform) + @property + def downloads(self): + """A dictionary of configuration information for downloading distros. + + The key "distros" should contain a list of `DistroFinder` instances similar + to "distro_paths". These are used to find and download distro versions. + + The key "cache_root" contains the Path to the directory where remote files + are downloaded for installation. + + The key "install_root" indicates where distros are installed. This should + normally be one of the "distro_paths" but should not contain glob wildcards. + """ + if self._downloads_parsed: + return self["downloads"] + + self._downloads_parsed = True + downloads = self.setdefault("downloads", {}) + + # Convert distros data into DistroFinder classes + distros = [] + for distro_finder in downloads.get("distros", []): + inst = self.entry_point_init( + "hab.download.finder", distro_finder[0], distro_finder[1:] + ) + # Ensure these items can access the site and its cache + inst.site = self + distros.append(inst) + downloads["distros"] = distros + + # Configure the download cache directory + cache_root = utils.Platform.default_download_cache() + if downloads.get("cache_root"): + # Use cache_root if its set to a non-empty value + paths = utils.Platform.expand_paths(downloads["cache_root"]) + if paths: + cache_root = paths[0] + downloads["cache_root"] = cache_root + + if "install_root" in downloads: + downloads["install_root"] = Path(downloads["install_root"]) + + return self["downloads"] + def dump(self, verbosity=0, color=None, width=80): """Return a string of the properties and their values. @@ -80,9 +126,20 @@ def dump(self, verbosity=0, color=None, width=80): if color is None: color = self.get("colorize", True) + def dump_object(value, prop): + """Convert value and prop into text with correct settings.""" + return utils.dump_object( + value, label=f"{prop}: ", color=color, width=width, verbosity=verbosity + ) + def cached_fmt(path, cached): + if hasattr(path, "dump"): + # Provide information about the finder class used instead of + # a simple path. + path = path.dump(verbosity=verbosity, color=color, width=width) if not cached: return path + if color: return f"{path} {Fore.YELLOW}(cached){Style.RESET_ALL}" else: @@ -96,7 +153,13 @@ def cached_fmt(path, cached): cache_file = self.cache.site_cache_path(path) path = cached_fmt(path, cache_file.is_file()) hab_paths.append(str(path)) - site_ret = utils.dump_object({"HAB_PATHS": hab_paths}, color=color, width=width) + site_ret = utils.dump_object( + {"HAB_PATHS": hab_paths}, color=color, width=width, verbosity=verbosity + ) + + # Ensure lazy loaded code is run before dumping + self.downloads + # Include all of the resolved site configurations ret = [] for prop, value in self.items(): @@ -107,28 +170,43 @@ def cached_fmt(path, cached): prop, value, cache, include_path=False ): paths.append(cached_fmt(dirname, cached)) - txt = utils.dump_object( - paths, label=f"{prop}: ", color=color, width=width - ) + txt = dump_object(paths, prop) elif verbosity < 1 and isinstance(value, dict): # This is too complex for most site dumps, hide the details behind # a higher verbosity setting. - txt = utils.dump_object( - f"Dictionary keys: {len(value)}", - label=f"{prop}: ", - color=color, - width=width, - ) + txt = dump_object(f"Dictionary keys: {len(value)}", prop) else: - txt = utils.dump_object( - value, label=f"{prop}: ", color=color, width=width - ) + txt = dump_object(value, prop) ret.append(txt) ret = "\n".join(ret) return utils.dump_title("Dump of Site", f"{site_ret}\n{ret}", color=color) + def entry_point_init(self, group, value, args, name=""): + """Initialize an entry point with args and kwargs. + + Args: + group (str): The entry point group name. + value (str): The entry point value used to import and resolve the class. + args (list): A list of arguments to pass to the class on init. If the + last item in this list is a dict, then it is passed to the kwargs + of the class. If not specified then the kwarg `site` will be set + to self. + name (str, optional): The entry point name. + + Returns: + A initialized object defined by the inputs. + """ + ep = EntryPoint(name, value, group) + ep_cls = ep.load() + kwargs = {} + if args and isinstance(args[-1], dict): + kwargs = args.pop() + if "site" not in kwargs: + kwargs["site"] = self + return ep_cls(*args, **kwargs) + def entry_points_for_group( self, group, default=None, entry_points=None, omit_none=True ): @@ -147,10 +225,6 @@ def entry_points_for_group( then don't include an EntryPoint object for it in the return. This allows a second site file to disable a entry_point already set. """ - # Delay this import to when required. It's faster than pkg_resources but - # no need to pay the import price for it if you are not using it. - from importlib_metadata import EntryPoint - ret = [] # Use the site defined entry_points if an override dict wasn't provided if entry_points is None: @@ -203,9 +277,28 @@ def load(self): self.paths.insert(0, path) self.load_file(path) - # Convert config_paths and distro_paths to lists of Path objects + # Convert config_paths to lists of Path objects self["config_paths"] = utils.Platform.expand_paths(self["config_paths"]) - self["distro_paths"] = utils.Platform.expand_paths(self["distro_paths"]) + + # Convert distro_paths to DistroFinder instances + distro_paths = [] + + default_distro_finder = self.get("entry_points", {}).get( + "hab.distro.finder.default", "hab.distro_finders.distro_finder:DistroFinder" + ) + for distro_finder in self["distro_paths"]: + if isinstance(distro_finder, str): + # Handle simple folder paths by converting to the DistroFinder class + distro_finder = [default_distro_finder, distro_finder] + + inst = self.entry_point_init( + "hab.distro.finder", distro_finder[0], distro_finder[1:] + ) + # Ensure these items can access the site and its cache + inst.site = self + distro_paths.append(inst) + + self["distro_paths"] = distro_paths # Ensure any platform_path_maps are converted to pathlib objects. self.standardize_platform_path_maps() @@ -339,10 +432,3 @@ def config_paths(self, config_paths): "config_paths", config_paths, cache, "*.json" ): yield dirname, path - - def distro_paths(self, distro_paths): - cache = self.cache.distro_paths() - for dirname, path, _ in self.cache.iter_cache_paths( - "distro_paths", distro_paths, cache, "*/.hab.json" - ): - yield dirname, path diff --git a/hab/utils.py b/hab/utils.py index d524350..66f2404 100644 --- a/hab/utils.py +++ b/hab/utils.py @@ -6,6 +6,7 @@ import os import re import sys +import tempfile import textwrap import zlib from abc import ABC, abstractmethod @@ -13,7 +14,7 @@ from collections.abc import KeysView from contextlib import contextmanager from datetime import date, datetime -from pathlib import Path, PurePath +from pathlib import Path, PurePath, PureWindowsPath import colorama @@ -29,6 +30,20 @@ class _JsonException(BaseException): """Placeholder exception when pyjson5 is not used. Should never be raised""" +# cloudpathlib is an optional library, if installed then this will enable +# `dump_object` to print its full path instead of just the filename. +try: + from cloudpathlib import CloudPath as _CloudPath +except ImportError: + + class _CloudPath: + """Placeholder class because `cloudpathlib` is not importable. + Used for isinstance checking. + """ + + pass + + colorama.init() re_windows_single_path = re.compile(r"^([a-zA-Z]:[\\\/][^:;]+)$") @@ -109,7 +124,7 @@ def decode_freeze(txt): return json.loads(data) -def dump_object(obj, label="", width=80, flat_list=False, color=False): +def dump_object(obj, label="", width=80, flat_list=False, color=False, verbosity=0): """Recursively convert python objects into a human readable table string. Args: @@ -128,6 +143,7 @@ def dump_object(obj, label="", width=80, flat_list=False, color=False): item to be broken into multiple lines. color (bool, optional): Use ANSI escape character sequences to colorize the output of the text. + verbosity (int, optional): More information is shown with higher values. """ pad = " " * len(label) if label: @@ -140,7 +156,10 @@ def dump_object(obj, label="", width=80, flat_list=False, color=False): if isinstance(obj, (list, KeysView)): rows = [] obj = [ - dump_object(o, width=width, flat_list=flat_list, color=color) for o in obj + dump_object( + o, width=width, flat_list=flat_list, color=color, verbosity=verbosity + ) + for o in obj ] if flat_list: # combine as many list items as possible onto each line @@ -178,12 +197,16 @@ def dump_object(obj, label="", width=80, flat_list=False, color=False): width=width, flat_list=flat_list, color=color, + verbosity=verbosity, ) ) lbl = pad return "\n".join(rows) - elif isinstance(obj, PurePath): + elif isinstance(obj, (PurePath, _CloudPath)): return f"{label}{obj}" + elif hasattr(obj, "dump"): + # If the class implements a dump method, return its result + return obj.dump(verbosity=verbosity, color=color, width=width) elif hasattr(obj, "name"): # Likely HabBase objects return f"{label}{obj.name}" @@ -261,6 +284,24 @@ def encode_freeze(data, version=None, site=None): return f'v{version}:{data.decode("utf-8")}' +def glob_path(path): + """Process any wildcards in the provided `pathlib.Path` like instance. + + While a `pathlib.Path` can be created with a glob string you won't be able to + resolve the glob into files. This function breaks the path down to its top level + item and calls `Path.glob` on the remaining path. So `Path("/mnt/*/.hab.json")` + gets converted into `Path("/mnt").glob("*/.hab.json")`. + + The input path will be converted to a `pathlib.Path` object for this operation. + + Based on https://stackoverflow.com/a/51108375 + """ + # Strip the path into its parts removing the root of absolute paths. + parts = path.parts[1:] + # From the root run a glob search on all parts of the glob string + return Path(path.parts[0]).glob(str(Path(*parts))) + + class HabJsonEncoder(_json.JSONEncoder): """JsonEncoder class that handles non-supported objects like hab.NotSet.""" @@ -276,6 +317,39 @@ def default(self, obj): return _json.JSONEncoder.default(self, obj) +def _load_json(source, load_funct, *args, **kwargs): + """Work function that parses json and ensures any errors report the source. + + Args: + source (os.PathLike or str): The source of the json data. This is reported + in any raised exceptions. + load_funct (callable): A function called to parse the json data. Normally + this is `json.load` or `json.loads`. + *args: Arguments passed to `load_funct`. + *kwargs: Keyword arguments passed to `load_funct`. + + Raises: + FileNotFoundError: If filename is not pointing to a file that actually exists. + pyjson5.Json5Exception: If using pyjson5, the error raised due to invalid json. + ValueError: If not using pyjson5, the error raised due to invalid json. + """ + try: + return load_funct(*args, **kwargs) + # Include the filename in the traceback to make debugging easier + except _JsonException as e: + # pyjson5 is installed add filename to the traceback + if e.result is None: + # Depending on the exception result may be None, convert it + # into a empty dict so we can add the filename + e.args = e.args[:1] + ({},) + e.args[2:] + e.result["source"] = str(source) + raise e.with_traceback(sys.exc_info()[2]) from None + except ValueError as e: + # Using python's native json parser + msg = f'{e} Source("{source}")' + raise type(e)(msg, e.doc, e.pos).with_traceback(sys.exc_info()[2]) from None + + def load_json_file(filename): """Open and parse a json file. If a parsing error happens the file path is added to the exception to allow for easier debugging. @@ -295,24 +369,30 @@ def load_json_file(filename): raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), str(filename)) with filename.open() as fle: - try: - data = json.load(fle) - # Include the filename in the traceback to make debugging easier - except _JsonException as e: - # pyjson5 is installed add filename to the traceback - if e.result is None: - # Depending on the exception result may be None, convert it - # into a empty dict so we can add the filename - e.args = e.args[:1] + ({},) + e.args[2:] - e.result["filename"] = str(filename) - raise e.with_traceback(sys.exc_info()[2]) from None - except ValueError as e: - # Using python's native json parser - msg = f'{e} Filename("{filename}")' - raise type(e)(msg, e.doc, e.pos).with_traceback(sys.exc_info()[2]) from None + data = _load_json(filename, json.load, fle) return data +def loads_json(json_string, source): + """Open and parse a json string. If a parsing error happens the source file + path is added to the exception to allow for easier debugging. + + Args: + json_string (str): The json data to parse. + source (pathlib.Path): The location json_string was pulled from. This is + reported if any parsing errors happen. + + Returns: + The data stored in the json file. + + Raises: + FileNotFoundError: If filename is not pointing to a file that actually exists. + pyjson5.Json5Exception: If using pyjson5, the error raised due to invalid json. + ValueError: If not using pyjson5, the error raised due to invalid json. + """ + return _load_json(source, json.loads, json_string) + + def natural_sort(ls, key=None): """Sort a list in a more natural way by treating contiguous integers as a single number instead of processing each number individually. This function @@ -463,6 +543,14 @@ def default_ext(cls): """Returns the default file extension used on this platform.""" return cls._default_ext + @classmethod + def default_download_cache(cls): + """Path where download files are cached. + + This is used as the default location for `Site.downloads["cache_root"]`. + """ + return Path(tempfile.gettempdir()) / "hab_downloads" + @classmethod def expand_paths(cls, paths): """Converts path strings separated by ``cls.pathsep()`` and lists into @@ -581,12 +669,20 @@ def normalize_path(cls, path): This ensures that the drive letter is resolved consistently to uppercase. """ - # Don't change the case of relative or UNC paths - if not path.is_absolute() or ":" not in path.drive: + if ( + # Don't change the case of relative or UNC paths + not path.is_absolute() + or ":" not in path.drive + # We only need to modify absolute WindowsPaths here, `CloudPath.is_absolute` + # always returns True, but does not inherit from PureWindowsPath. + # This prevents `CloudPath.client` from getting lost by the recast below. + or not isinstance(path, PureWindowsPath) + ): return path + parts = path.parts - cls = type(path) - return cls(parts[0].upper()).joinpath(*parts[1:]) + path_cls = type(path) + return path_cls(parts[0].upper()).joinpath(*parts[1:]) @classmethod def pathsep(cls, ext=None, key=None): diff --git a/setup.cfg b/setup.cfg index 68ca54a..2392826 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ install_requires = importlib-metadata packaging>=20.0 setuptools-scm[toml]>=4 -python_requires = >=3.6 +python_requires = >=3.7 include_package_data = True scripts = bin/.hab-complete.bash @@ -50,6 +50,9 @@ scripts = exclude = tests [options.extras_require] +cloud = + cloudpathlib + remotezip dev = black==22.12.0 covdefaults @@ -62,6 +65,10 @@ dev = tox json5 = pyjson5 +s3 = + cloudpathlib[s3] + remotezip + requests-aws4auth [flake8] select = B, C, E, F, N, W, B9 diff --git a/tests/conftest.py b/tests/conftest.py index dab4f29..2aed682 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,13 @@ import json import os +import shutil +from collections import namedtuple from contextlib import contextmanager from pathlib import Path, PurePath +from zipfile import ZipFile import pytest +from jinja2 import Environment, FileSystemLoader from packaging.requirements import Requirement from hab import Resolver, Site @@ -111,6 +115,110 @@ def resolver(request): return request.getfixturevalue(test_map[request.param]) +Distro = namedtuple("Distro", ["name", "version", "inc_version", "distros"]) + + +class DistroInfo(namedtuple("DistroInfo", ["root", "versions"])): + default_versions = ( + ("dist_a", "0.1", True, None), + ("dist_a", "0.2", False, ["dist_b"]), + ("dist_a", "1.0", False, None), + ("dist_b", "0.5", False, None), + ("dist_b", "0.6", False, None), + ) + + @classmethod + def dist_version(cls, distro, version): + return f"{distro}_v{version}" + + @classmethod + def hab_json(cls, distro, version=None, distros=None): + data = {"name": distro} + if version: + data["version"] = version + if distros: + data["distros"] = distros + return json.dumps(data, indent=4) + + @classmethod + def generate(cls, root, versions=None, zip_created=None): + if versions is None: + versions = cls.default_versions + + versions = {(x[0], x[1]): Distro(*x) for x in versions} + + for version in versions.values(): + name = cls.dist_version(version.name, version.version) + filename = root / f"{name}.zip" + ver = version.version if version.inc_version else None + with ZipFile(filename, "w") as zf: + zf.writestr( + ".hab.json", + cls.hab_json(version.name, version=ver, distros=version.distros), + ) + zf.writestr("file_a.txt", "File A inside the distro.") + zf.writestr("folder/file_b.txt", "File B inside the distro.") + if zip_created: + zip_created(zf) + + # Create a correctly named .zip file that doesn't have a .hab.json file + # to test for .zip files that are not distros. + with ZipFile(root / "not_valid_v0.1.zip", "w") as zf: + zf.writestr("README.txt", "This file is not a hab distro zip.") + + return cls(root, versions) + + +@pytest.fixture(scope="session") +def distro_finder_info(tmp_path_factory): + """Returns a DistroInfo instance with extracted distros ready for hab. + + This is useful for using an existing hab distro structure as your download server. + """ + root = tmp_path_factory.mktemp("_distro_finder") + + def zip_created(zf): + """Extract all contents zip into a distro folder structure.""" + filename = Path(zf.filename).stem + distro, version = filename.split("_v") + zf.extractall(root / distro / version) + + return DistroInfo.generate(root, zip_created=zip_created) + + +@pytest.fixture(scope="session") +def zip_distro(tmp_path_factory): + """Returns a DistroInfo instance for a zip folder structure. + + This is useful if the zip files are locally accessible or if your hab download + server supports `HTTP range requests`_. For example if you are using Amazon S3. + + .. _HTTP range requests: + https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests + """ + root = tmp_path_factory.mktemp("_zip_distro") + return DistroInfo.generate(root) + + +@pytest.fixture(scope="session") +def zip_distro_sidecar(tmp_path_factory): + """Returns a DistroInfo instance for a zip folder structure with sidecar + `.hab.json` files. + + This is useful when your hab download server does not support HTTP range requests. + """ + root = tmp_path_factory.mktemp("_zip_distro_sidecar") + + def zip_created(zf): + """Extract the .hab.json from the zip to a sidecar file.""" + filename = Path(zf.filename).stem + sidecar = root / f"{filename}.hab.json" + path = zf.extract(".hab.json", root) + shutil.move(path, sidecar) + + return DistroInfo.generate(root, zip_created=zip_created) + + class Helpers(object): """A collection of reusable functions that tests can use.""" @@ -204,6 +312,36 @@ def compare_files(generated, check): cache[i] == check[i] ), f"Difference on line: {i} between the generated cache and {generated}." + @staticmethod + def render_template(template, dest, **kwargs): + """Render a jinja template in from the test templates directory. + + Args: + template (str): The name of the template file in the templates dir. + dest (os.PathLike): The destination filename to write the output. + **kwargs: All kwargs are used to render the template. + """ + environment = Environment( + loader=FileSystemLoader(str(Path(__file__).parent / "templates")), + trim_blocks=True, + lstrip_blocks=True, + ) + template = environment.get_template(template) + + text = template.render(**kwargs).rstrip() + "\n" + with dest.open("w") as fle: + fle.write(text) + + @classmethod + def render_resolver(cls, site_template, dest, **kwargs): + """Calls `render_template` and constructs a Resolver instance for it.""" + # Build the hab site + site_file = dest / "site.json" + cls.render_template(site_template, site_file, **kwargs) + + site = Site([site_file]) + return Resolver(site) + @pytest.fixture def helpers(): diff --git a/tests/site/site_distro_finder.json b/tests/site/site_distro_finder.json new file mode 100644 index 0000000..414caee --- /dev/null +++ b/tests/site/site_distro_finder.json @@ -0,0 +1,32 @@ +{ + "set": + { + "distro_paths": + [ + [ + "hab.distro_finders.distro_finder:DistroFinder", + "hab testable/download/path" + ], + [ + "hab.distro_finders.distro_finder:DistroFinder", + "hab testing/downloads", + { + "site": "for testing only, do not specify site" + } + ] + ], + "downloads": + { + "cache_root": "hab testable/download/path", + "distros": + [ + [ + "hab.distro_finders.df_zip:DistroFinderZip", + "network_server/distro/source" + ] + ], + "install_root": "{relative_root}/distros", + "relative_path": "{{distro_name}}_v{{version}}" + } + } +} diff --git a/tests/site/site_distro_finder_empty.json b/tests/site/site_distro_finder_empty.json new file mode 100644 index 0000000..240abbe --- /dev/null +++ b/tests/site/site_distro_finder_empty.json @@ -0,0 +1,7 @@ +{ + "set": { + "downloads": { + "cache_root": "" + } + } +} diff --git a/tests/templates/site_distro_finder.json b/tests/templates/site_distro_finder.json new file mode 100644 index 0000000..a0b7c7a --- /dev/null +++ b/tests/templates/site_distro_finder.json @@ -0,0 +1,20 @@ +{ + "set": { + "config_paths": [ + "{relative_root}/configs" + ], + "distro_paths": [ + "{relative_root}/distros/*" + ], + "downloads": { + "cache_root": "{relative_root}/downloads", + "distros": [ + [ + "hab.distro_finders.distro_finder:DistroFinder", + "{{ zip_root }}/*" + ] + ], + "install_root": "{relative_root}/distros" + } + } +} diff --git a/tests/templates/site_distro_zip.json b/tests/templates/site_distro_zip.json new file mode 100644 index 0000000..b4fd30e --- /dev/null +++ b/tests/templates/site_distro_zip.json @@ -0,0 +1,20 @@ +{ + "set": { + "config_paths": [ + "{relative_root}/configs" + ], + "distro_paths": [ + "{relative_root}/distros/*" + ], + "downloads": { + "cache_root": "{relative_root}/downloads", + "distros": [ + [ + "hab.distro_finders.df_zip:DistroFinderZip", + "{{ zip_root }}" + ] + ], + "install_root": "{relative_root}/distros" + } + } +} diff --git a/tests/templates/site_distro_zip_sidecar.json b/tests/templates/site_distro_zip_sidecar.json new file mode 100644 index 0000000..689cd83 --- /dev/null +++ b/tests/templates/site_distro_zip_sidecar.json @@ -0,0 +1,20 @@ +{ + "set": { + "config_paths": [ + "{relative_root}/configs" + ], + "distro_paths": [ + "{relative_root}/distros/*" + ], + "downloads": { + "cache_root": "{relative_root}/downloads", + "distros": [ + [ + "hab.distro_finders.zip_sidecar:DistroFinderZipSidecar", + "{{ zip_root }}" + ] + ], + "install_root": "{relative_root}/distros" + } + } +} diff --git a/tests/templates/site_download.json b/tests/templates/site_download.json new file mode 100644 index 0000000..b4fd30e --- /dev/null +++ b/tests/templates/site_download.json @@ -0,0 +1,20 @@ +{ + "set": { + "config_paths": [ + "{relative_root}/configs" + ], + "distro_paths": [ + "{relative_root}/distros/*" + ], + "downloads": { + "cache_root": "{relative_root}/downloads", + "distros": [ + [ + "hab.distro_finders.df_zip:DistroFinderZip", + "{{ zip_root }}" + ] + ], + "install_root": "{relative_root}/distros" + } + } +} diff --git a/tests/test_distro_finder.py b/tests/test_distro_finder.py new file mode 100644 index 0000000..2b8fcc1 --- /dev/null +++ b/tests/test_distro_finder.py @@ -0,0 +1,398 @@ +import glob +import logging +from pathlib import Path + +import pytest + +from hab import DistroMode, Resolver, Site, utils +from hab.distro_finders import df_zip, distro_finder, zip_sidecar +from hab.errors import InstallDestinationExistsError +from hab.parsers import DistroVersion + + +def test_distro_finder_entry_point(config_root): + """Test edge cases for DistroFinder entry_point processing.""" + paths = [config_root / "site" / "site_distro_finder.json"] + site = Site(paths) + distro_paths = site["distro_paths"] + # Ensure the DistroFinder paths are set correctly when set as EntryPoint + assert distro_paths[0].root == Path("hab testable") / "download" / "path" + assert distro_paths[1].root == Path("hab testing") / "downloads" + # The second path passes the kwargs dict with `site`. This triggers testing + # when a dict is passed to the entry_point. However site is always set to + # the current site after a DistroFinder is initialized. + assert distro_paths[1].site == site + + +def test_eq(): + a = distro_finder.DistroFinder("path/a") + + assert a == distro_finder.DistroFinder("path/a") + assert a != distro_finder.DistroFinder("path/b") + + # Test that if the glob_str is different it will not compare equal + b = distro_finder.DistroFinder("path/a") + b.glob_str = "*/test.json" + assert a != b + # Test that if glob_str attr is missing it will not compare equal + del b.glob_str + assert a != b + # Restore glob_str and the objects will compare equal again + b.glob_str = "*/.hab.json" + assert a == b + + # Test that if the root is different it will not compare equal + b.root = Path(".") + assert a != b + # Test that if root attr is missing it will not compare equal + del b.root + assert a != b + # Restore root and the objects will compare equal again + b.root = Path("path/a") + assert a == b + + +@pytest.mark.parametrize( + "glob_str,count", + ( + ("{root}/reference*/sh_*", 12), + ("{root}/reference/*", 0), + ("{root}/reference_scripts/*/*.sh", 20), + ), +) +def test_glob_path(config_root, glob_str, count): + """Ensure `hab.utils.glob_path` returns the expected results.""" + glob_str = glob_str.format(root=config_root) + # Check against the `glob.glob` result. + check = sorted([Path(p) for p in glob.glob(glob_str)]) + + path_with_glob = Path(glob_str) + result = sorted(utils.glob_path(path_with_glob)) + + assert result == check + # Sanity check to ensure that the expected results were found by `glob.glob` + assert len(result) == count + + +class TestLoadPath: + """Test the various `DistroFinder.load_path` implementations.""" + + def test_distro_finder(self, uncached_resolver): + """Currently load_path for DistroFinder just returns None.""" + finder = distro_finder.DistroFinder("", uncached_resolver.site) + assert finder.load_path(Path(".")) is None + + def test_zip_sidecar(self, zip_distro_sidecar): + """The Zip Sidecar reads a .json file next to the zip distro. + + Ensure it's able to read data from the .json file. + """ + finder = zip_sidecar.DistroFinderZipSidecar(zip_distro_sidecar.root) + + # This distro hard codes the version inside the .json file + data = finder.load_path(zip_distro_sidecar.root / "dist_a_v0.1.hab.json") + assert data["name"] == "dist_a" + assert "distros" not in data + assert data["version"] == "0.1" + + # Test a different distro that doesn't hard code the version + data = finder.load_path(zip_distro_sidecar.root / "dist_b_v0.5.hab.json") + assert data["name"] == "dist_b" + assert "distros" not in data + assert data["version"] == "0.5" + + # This distro includes required distros + data = finder.load_path(zip_distro_sidecar.root / "dist_a_v0.2.hab.json") + assert data["name"] == "dist_a" + assert data["distros"] == ["dist_b"] + assert data["version"] == "0.2" + + def test_s3(self): + pass + + +class CheckDistroFinder: + distro_finder_cls = distro_finder.DistroFinder + site_template = "site_distro_finder.json" + + def create_resolver(self, zip_root, helpers, tmp_path): + """Create a hab site for the test.""" + return helpers.render_resolver( + self.site_template, tmp_path, zip_root=zip_root.as_posix() + ) + + def check_installed(self, a_distro_finder, helpers, tmp_path): + resolver = self.create_resolver(a_distro_finder.root, helpers, tmp_path) + finder = resolver.distro_paths[0] + distro_folder = resolver.site.downloads["install_root"] / "dist_a" / "0.1" + + # The distro is not installed yet + assert not distro_folder.exists() + assert not finder.installed(distro_folder) + + # Simulate installing by creating the .hab.json file(contents doesn't matter) + distro_folder.mkdir(parents=True) + with (distro_folder / ".hab.json").open("w"): + pass + assert finder.installed(distro_folder) + + def check_install(self, a_distro_finder, helpers, tmp_path): + resolver = self.create_resolver(a_distro_finder.root, helpers, tmp_path) + dl_finder = resolver.site.downloads["distros"][0] + assert isinstance(dl_finder, self.distro_finder_cls) + install_root = resolver.site.downloads["install_root"] + + for di in a_distro_finder.versions.values(): + # Get the downloadable distro + with resolver.distro_mode_override(DistroMode.Downloaded): + dl_distro = resolver.find_distro(f"{di.name}=={di.version}") + + # Ensure the finder used to create this distro is set + assert dl_distro.finder == dl_finder + + dest = install_root / dl_distro.distro_name / str(dl_distro.version) + assert not dest.exists() + dl_finder.install(dl_distro.filename, dest) + assert dest.is_dir() + assert (dest / ".hab.json").exists() + assert (dest / "file_a.txt").exists() + assert (dest / "folder/file_b.txt").exists() + + # Test that if you try to install an already existing distro + # an exception is raised + with pytest.raises( + InstallDestinationExistsError, match="The destination already exists:" + ) as excinfo: + dl_finder.install(dl_distro.filename, dest) + assert excinfo.value.filename == dest + + +class TestDistroFinder(CheckDistroFinder): + distro_finder_cls = distro_finder.DistroFinder + site_template = "site_distro_finder.json" + + def test_content(self, distro_finder_info): + """Content always returns the parent of the provided path currently.""" + finder = self.distro_finder_cls(distro_finder_info.root) + # We may want to improve this later, but it works for now + path = distro_finder_info.root / ".hab.json" + result = finder.content(path) + assert result == distro_finder_info.root + + def test_installed(self, distro_finder_info, helpers, tmp_path): + self.check_installed(distro_finder_info, helpers, tmp_path) + + def test_install(self, distro_finder_info, helpers, tmp_path): + self.check_install(distro_finder_info, helpers, tmp_path) + + +class TestZipSidecar(CheckDistroFinder): + """Tests specific to `DistroFinderZip`.""" + + distro_finder_cls = zip_sidecar.DistroFinderZipSidecar + site_template = "site_distro_zip_sidecar.json" + + def test_installed(self, zip_distro_sidecar, helpers, tmp_path): + self.check_installed(zip_distro_sidecar, helpers, tmp_path) + + def test_install(self, zip_distro_sidecar, helpers, tmp_path): + self.check_install(zip_distro_sidecar, helpers, tmp_path) + + +class TestZip(CheckDistroFinder): + """Tests specific to `DistroFinderZip`.""" + + distro_finder_cls = df_zip.DistroFinderZip + site_template = "site_distro_zip.json" + + def test_content(self, zip_distro): + finder = df_zip.DistroFinderZip(zip_distro.root) + # If path is already a .zip file, it is just returned + path = zip_distro.root / "already_zip.zip" + result = finder.content(path) + assert result == path + + # The right most .zip file is returned if path has multiple .zip suffixes. + path = zip_distro.root / "a.zip" / "b.zip" + result = finder.content(path) + assert result == path + + # If a member path is passed, return the right most .zip suffix. + member_path = path / ".hab.json" + result = finder.content(member_path) + assert result == path + + # member paths with nested return the right most .zip suffix. + member_path = path / "folder" / "sub-folder" / "file.json" + result = finder.content(member_path) + assert result == path + + # If no .zip suffix is passed, the original path is returned. + path = zip_distro.root / "not_an_archive.txt" + result = finder.content(path) + assert result == path + + def test_load_path(self, zip_distro): + """The Zip finder reads a .json file from inside the zip distro file. + + Ensure it's able to read data from the .json file. + """ + finder = df_zip.DistroFinderZip(zip_distro.root) + + # This distro hard codes the version inside the .json file + data = finder.load_path(zip_distro.root / "dist_a_v0.1.zip") + assert data["name"] == "dist_a" + assert "distros" not in data + assert data["version"] == "0.1" + + # Test a different distro that doesn't hard code the version + data = finder.load_path(zip_distro.root / "dist_b_v0.5.zip") + assert data["name"] == "dist_b" + assert "distros" not in data + assert data["version"] == "0.5" + + # This distro includes required distros + data = finder.load_path(zip_distro.root / "dist_a_v0.2.zip") + assert data["name"] == "dist_a" + assert data["distros"] == ["dist_b"] + assert data["version"] == "0.2" + + def test_zip_get_file_data(self, zip_distro, caplog): + """Test edge cases for `DistroFinderZip.get_file_data`.""" + finder = df_zip.DistroFinderZip(zip_distro.root) + assert finder._cache == {} + + # This file doesn't have a .hab.json file inside it + path = zip_distro.root / "not_valid_v0.1.zip" + data = finder.get_file_data(path) + assert data is None + assert [path / ".hab.json"] == list(finder._cache.keys()) + finder.clear_cache() + + # Check what happens if a member path isn't provided(Just the .zip file path) + path = zip_distro.root / "dist_a_v0.1.zip" + member_path = path / ".hab.json" + caplog.clear() + with caplog.at_level(logging.DEBUG, logger="hab.distro_finders.df_zip"): + data = finder.get_file_data(path) + check = [f'Implicitly added member ".hab.json" to path "{member_path}".'] + assert check == [rec.message for rec in caplog.records] + # The raw data text was read and returned + assert data == b'{\n "name": "dist_a",\n "version": "0.1"\n}' + assert member_path in finder._cache + + # Test that the cache is returned if populated + data = "Data already in the cache" + finder._cache[Path(member_path)] = data + assert finder.get_file_data(member_path) is data + + def test_installed(self, zip_distro, helpers, tmp_path): + self.check_installed(zip_distro, helpers, tmp_path) + + def test_install(self, zip_distro, helpers, tmp_path): + self.check_install(zip_distro, helpers, tmp_path) + + +# TODO: Break this into separate smaller tests of components for each class not this +@pytest.mark.parametrize( + "distro_info", + ( + # "distro_finder_info", + "zip_distro", + "zip_distro_sidecar", + ), +) +def dtest_zip(request, distro_info, helpers, tmp_path): + # Convert the distro_info parameter to testing values. + df_cls = df_zip.DistroFinderZip + hab_json = ".hab.json" + implements_cache = True + parent_type = True + site_filename = "site_distro_zip.json" + if distro_info == "zip_distro_sidecar": + df_cls = zip_sidecar.DistroFinderZipSidecar + hab_json = "{name}_v{ver}.hab.json" + implements_cache = False + parent_type = "sidecar" + site_filename = "site_distro_zip_sidecar.json" + elif distro_info == "distro_finder_info": + df_cls = distro_finder.DistroFinder + implements_cache = False + parent_type = "directory" + site_filename = "site_distro_finder.json" + distro_info = request.getfixturevalue(distro_info) + + site_file = tmp_path / "site.json" + helpers.render_template( + site_filename, site_file, zip_root=distro_info.root.as_posix() + ) + site_distros = tmp_path / "distros" + + check = set([v[:2] for v in distro_info.versions]) + + site = Site([site_file]) + resolver = Resolver(site) + results = set() + # The correct class was resolved + df = resolver.distro_paths[0] + assert type(df) == df_cls + + if implements_cache: + assert df._cache == {} + + for node in resolver.dump_forest(resolver.distros, attr=None): + distro = node.node + if not isinstance(distro, DistroVersion): + continue + + # Ensure the finder used to create this distro is set + assert distro.finder == df + + assert distro.filename.name == hab_json.format( + name=distro.distro_name, ver=distro.version + ) + if parent_type == "zip": + # If the parent is a zip, then the parent is a zip file + assert distro.filename.parent.suffix == ".zip" + assert distro.filename.parent.is_file() + elif parent_type == "sidecar": + # There is a sidecar zip file next to the *.hab.json file + zip_filename = distro.filename.name.replace(".hab.json", ".zip") + assert (distro.filename.parent / zip_filename).is_file() + elif parent_type == "directory": + assert distro.filename.is_file() + assert distro.filename.name == ".hab.json" + + if implements_cache: + assert distro.filename in df._cache + + results.add((distro.distro_name, str(distro.version))) + + # Test the install process extracts all of the files from the zip + dest = site_distros / distro.distro_name / str(distro.version) + assert not dest.exists() + df.install(distro.filename, dest) + assert dest.is_dir() + assert (dest / ".hab.json").exists() + assert (dest / "file_a.txt").exists() + assert (dest / "folder/file_b.txt").exists() + + # Test that if you try to install an already existing distro + # an exception is raised + with pytest.raises( + InstallDestinationExistsError, match="The destination already exists:" + ) as excinfo: + df.install(distro.filename, dest) + assert excinfo.value.filename == dest + + # Test the installed function + # Returns True if passed a distro version folder containing a .hab.json + assert df.installed(dest) + # It returns False if the .hab.json file doesn't exist + assert not df.installed(site_distros) + + if implements_cache: + df.clear_cache() + assert df._cache == {} + + assert results == check diff --git a/tests/test_launch.py b/tests/test_launch.py index a0b2a88..fda8429 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -220,7 +220,7 @@ class TestCliExitCodes: @pytest.mark.skipif(sys.platform != "win32", reason="only applies on windows") @missing_annotations_hack - def test_bat(self, config_root, exit_code): + def test_bat(self, config_root, exit_code, tmp_path): hab_bin = (config_root / ".." / "bin" / "hab.bat").resolve() # fmt: off cmd = [ @@ -230,8 +230,13 @@ def test_bat(self, config_root, exit_code): ] # fmt: on + # When running tox in parallel we may run into the `%RANDOM%` collision. + # Change the `TMP` env var to a per-test unique folder to avoid this. + env = os.environ.copy() + env["TMP"] = str(tmp_path) + # Run the hab command in a subprocess - proc = subprocess.run(cmd, **self.run_kwargs) + proc = subprocess.run(cmd, env=env, **self.run_kwargs) # Check that the print statement was actually run assert proc.stdout == self.output_text diff --git a/tests/test_lazy_distro_version.py b/tests/test_lazy_distro_version.py new file mode 100644 index 0000000..0745591 --- /dev/null +++ b/tests/test_lazy_distro_version.py @@ -0,0 +1,151 @@ +import pytest +from packaging.requirements import Requirement +from packaging.version import Version + +from hab import DistroMode +from hab.errors import InstallDestinationExistsError +from hab.parsers.lazy_distro_version import DistroPath, LazyDistroVersion + + +def test_distro_path(zip_distro_sidecar, helpers, tmp_path): + resolver = helpers.render_resolver( + "site_distro_zip_sidecar.json", + tmp_path, + zip_root=zip_distro_sidecar.root.as_posix(), + ) + with resolver.distro_mode_override(DistroMode.Downloaded): + distro = resolver.find_distro("dist_a==0.2") + + # Passing root as a string converts it to a pathlib.Path object. + dpath = DistroPath( + distro, str(tmp_path), relative="{distro_name}-v{version}", site=resolver.site + ) + # Test that the custom relative string, it used to generate root + assert dpath.root == tmp_path / "dist_a-v0.2" + assert dpath.hab_filename == tmp_path / "dist_a-v0.2" / ".hab.json" + + # If site and relative are not passed the default is used + dpath = DistroPath(distro, tmp_path) + assert dpath.root == tmp_path / "dist_a" / "0.2" + assert dpath.hab_filename == tmp_path / "dist_a" / "0.2" / ".hab.json" + + # Test that site settings are respected when not passing relative + resolver.site.downloads["relative_path"] = "parent/{distro_name}/child/{version}" + dpath = DistroPath(distro, tmp_path, site=resolver.site) + assert dpath.root == tmp_path / "parent" / "dist_a" / "child" / "0.2" + assert ( + dpath.hab_filename + == tmp_path / "parent" / "dist_a" / "child" / "0.2" / ".hab.json" + ) + + +def test_is_lazy(zip_distro_sidecar, helpers, tmp_path): + """Check that a LazyDistroVersion doesn't automatically load all data.""" + resolver = helpers.render_resolver( + "site_distro_zip_sidecar.json", + tmp_path, + zip_root=zip_distro_sidecar.root.as_posix(), + ) + with resolver.distro_mode_override(DistroMode.Downloaded): + distro = resolver.find_distro("dist_a==0.1") + + frozen_data = dict( + context=["dist_a"], + name="dist_a==0.1", + version=Version("0.1"), + ) + filename = zip_distro_sidecar.root / "dist_a_v0.1.hab.json" + + # The find_distro call should have called load but does not actually load data + assert isinstance(distro, LazyDistroVersion) + assert distro._loaded is False + assert distro.context == ["dist_a"] + assert distro.filename == filename + assert distro.frozen_data == frozen_data + assert distro.name == "dist_a==0.1" + + # Calling _ensure_loaded actually loads the full distro from the finder's data + data = distro._ensure_loaded() + assert distro._loaded is True + assert isinstance(data, dict) + assert distro.name == "dist_a==0.1" + + # If called a second time, then nothing extra is done and no data is returned. + assert distro._ensure_loaded() is None + + +def test_bad_kwargs(): + """Test that the proper error is raised if you attempt to init with a filename.""" + match = "Passing filename to this class is not supported." + with pytest.raises(ValueError, match=match): + LazyDistroVersion(None, None, "filename") + + with pytest.raises(ValueError, match=match): + LazyDistroVersion(None, None, filename="a/filename") + + +@pytest.mark.parametrize( + "prop,check", + (("distros", {"dist_b": Requirement("dist_b")}),), +) +def test_lazy_hab_property(prop, check, zip_distro_sidecar, helpers, tmp_path): + """Check that a LazyDistroVersion doesn't automatically load all data.""" + resolver = helpers.render_resolver( + "site_distro_zip_sidecar.json", + tmp_path, + zip_root=zip_distro_sidecar.root.as_posix(), + ) + with resolver.distro_mode_override(DistroMode.Downloaded): + distro = resolver.find_distro("dist_a==0.2") + + # Calling a lazy getter ensures the data is loaded + assert distro._loaded is False + value = getattr(distro, prop) + assert distro._loaded is True + assert value == check + + # You can call the lazy getter repeatedly + value = getattr(distro, prop) + assert value == check + + +def test_install(zip_distro_sidecar, helpers, tmp_path): + """Check that a LazyDistroVersion doesn't automatically load all data.""" + resolver = helpers.render_resolver( + "site_distro_zip_sidecar.json", + tmp_path, + zip_root=zip_distro_sidecar.root.as_posix(), + ) + with resolver.distro_mode_override(DistroMode.Downloaded): + distro = resolver.find_distro("dist_a==0.2") + dest_root = resolver.site.downloads["install_root"] + distro_root = dest_root / "dist_a" / "0.2" + hab_json = distro_root / ".hab.json" + + # The distro is not currently installed. This also tests that it can + # auto-cast to DistroPath + assert not distro.installed(dest_root) + + # Install will clear the cache, ensure its populated + assert resolver._downloadable_distros is not None + # Install the distro using LazyDistroVersion + distro.install(dest_root) + assert distro.installed(dest_root) + assert hab_json.exists() + # Check that the cache was cleared by the install function + assert resolver._downloadable_distros is None + + # Test that if the distro is already installed, an error is raised + with pytest.raises(InstallDestinationExistsError) as excinfo: + distro.install(dest_root) + assert excinfo.value.filename == distro_root + + # Test forced replacement of an existing distro by creating an extra file + extra_file = distro_root / "extra_file.txt" + extra_file.touch() + # This won't raise the exception, but will remove the old distro + distro.install(dest_root, replace=True) + assert hab_json.exists() + assert distro.installed(dest_root) + + assert not extra_file.exists() diff --git a/tests/test_parsing.py b/tests/test_parsing.py index 4d81378..0be9f17 100644 --- a/tests/test_parsing.py +++ b/tests/test_parsing.py @@ -1,7 +1,9 @@ import copy import json +import pickle import re import sys +from datetime import date, datetime from pathlib import Path import anytree @@ -9,7 +11,8 @@ import setuptools_scm from packaging.version import Version -from hab import NotSet, utils +from hab import NotSet, Resolver, utils +from hab.distro_finders.distro_finder import DistroFinder from hab.errors import ( DuplicateJsonError, HabError, @@ -17,10 +20,10 @@ ReservedVariableNameError, _IgnoredVersionError, ) -from hab.parsers import Config, DistroVersion, FlatConfig +from hab.parsers import Config, DistroVersion, FlatConfig, HabBase -class TestLoadJsonFile: +class TestLoadJson: """Tests various conditions when using `hab.utils.load_json_file` to ensure expected output. """ @@ -34,16 +37,28 @@ def test_missing(self, tmpdir): utils.load_json_file(path) assert Path(excinfo.value.filename) == path + @classmethod + def check_exception(cls, excinfo, native_json, path): + if native_json: + # If built-in json was used, check that filename was appended to the message + assert f'Source("{path}")' in str(excinfo.value) + else: + # If pyjson5 was used, check that the filename was added to str + assert f"'source': {str(path)!r}" in str(excinfo.value) + # Check that the filename was added to the result dict + assert excinfo.value.result["source"] == str(path) + def test_binary(self, tmpdir): """If attempting to read a binary file, filename is included in exception. This is a problem we run into rarely where a text file gets replaced/generated with a binary file containing noting but a lot of null bytes. """ + bin_data = b"\x00" * 32 path = Path(tmpdir) / "binary.json" # Create a binary test file containing multiple binary null values. with path.open("wb") as fle: - fle.write(b"\x00" * 32) + fle.write(bin_data) # Detect if using pyjson5 or not native_json = False @@ -56,15 +71,15 @@ def test_binary(self, tmpdir): else: exc_type = pyjson5.pyjson5.Json5IllegalCharacter + # Test load_json_file with pytest.raises(exc_type) as excinfo: utils.load_json_file(path) + self.check_exception(excinfo, native_json, path) - if native_json: - # If built-in json was used, check that filename was appended to the message - assert f'Filename("{path}")' in str(excinfo.value) - else: - # If pyjson5 was used, check that the filename was added to the result dict - assert f"{{'filename': {str(path)!r}}}" in str(excinfo.value) + # Test loads_json + with pytest.raises(exc_type) as excinfo: + utils.loads_json(bin_data.decode(), path) + self.check_exception(excinfo, native_json, path) def test_config_load(self, uncached_resolver): cfg = Config({}, uncached_resolver) @@ -77,6 +92,18 @@ def test_config_load(self, uncached_resolver): with pytest.raises(FileNotFoundError): cfg.load("invalid_path.json") + def test_loads_json(self, config_root): + """Test that `loads_json` is able to parse a valid json string.""" + filename = config_root / "site_main.json" + with filename.open() as fle: + text = fle.read() + # Test an existing file is able to be parsed successfully. + data = utils.loads_json(text, filename) + # Spot check that we were able to parse data from the file. + assert isinstance(data, dict) + assert "append" in data + assert "set" in data + def test_distro_parse(config_root, resolver): """Check that a distro json can be parsed correctly""" @@ -182,13 +209,18 @@ def get_ver(*args, **kwargs): app.load(path) -def test_distro_version(resolver): +def test_distro_version(resolver, zip_distro_sidecar): """Verify that we find the expected version for a given requirement.""" maya = resolver.distros["maya2020"] assert maya.latest_version("maya2020").name == "maya2020==2020.1" assert maya.latest_version("maya2020<2020.1").name == "maya2020==2020.0" + forest = {} + resolver = Resolver() + parsed = HabBase(forest, resolver, zip_distro_sidecar.root / "dist_a_v0.1.hab.json") + assert parsed.version == Version("0.1") + def test_config_parse(config_root, resolver, helpers): """Check that a config json can be parsed correctly""" @@ -291,6 +323,7 @@ def test_metaclass(): "environment", "environment_config", "filename", + "finder", "min_verbosity", "name", "optional_distros", @@ -686,19 +719,16 @@ def test_invalid_config(config_root, resolver): with pytest.raises(_JsonException) as excinfo: Config({}, resolver, filename=path) - - if native_json: - # If built-in json was used, check that filename was appended to the message - assert f'Filename("{path}")' in str(excinfo.value) - else: - # If pyjson5 was used, check that the filename was added to the result dict - assert excinfo.value.result["filename"] == str(path) + TestLoadJson.check_exception(excinfo, native_json, path) def test_misc_coverage(resolver): """Test that cover misc lines not covered by the rest of the tests""" assert str(NotSet) == "NotSet" assert copy.copy(NotSet) is NotSet + # Check that NotSet can be pickled + payload = pickle.dumps(NotSet) + assert pickle.loads(payload) is NotSet # Check that dirname is modified when setting a blank filename cfg = Config({}, resolver) @@ -761,10 +791,13 @@ def test_duplicated_distros(config_root, resolver): definitions are in the same config_path so a DuplicateJsonError is raised. """ original = resolver.distro_paths + site = resolver.site # Check that the first config in distro_paths was used distro_paths = list(original) - distro_paths.insert(0, config_root / "duplicates" / "distros_1" / "*") + distro_paths.insert( + 0, DistroFinder(config_root / "duplicates" / "distros_1" / "*", site=site) + ) resolver.distro_paths = distro_paths dcc = resolver.find_distro("the_dcc==1.2") @@ -774,7 +807,9 @@ def test_duplicated_distros(config_root, resolver): # Check that an exception is raised if there are duplicate definitions from # the same distro_paths directory. distro_paths = list(original) - distro_paths.insert(0, config_root / "duplicates" / "distros_2" / "*") + distro_paths.insert( + 0, DistroFinder(config_root / "duplicates" / "distros_2" / "*", site=site) + ) resolver.distro_paths = distro_paths with pytest.raises(DuplicateJsonError): @@ -1172,7 +1207,7 @@ def test_reserved(self, uncached_resolver, variables, invalid, tmpdir): # Add the test distro to hab's distro search. We don't need to call # `clear_caches` because distros haven't been resolved yet. - uncached_resolver.distro_paths.append(Path(tmpdir)) + uncached_resolver.distro_paths.append(DistroFinder(Path(tmpdir))) # When distros are resolved, an exception should be raised with pytest.raises( @@ -1180,3 +1215,14 @@ def test_reserved(self, uncached_resolver, variables, invalid, tmpdir): match=rf"'{invalid}' are reserved variable name\(s\) for hab", ): uncached_resolver.distros + + +def test_hab_json_encoder(): + # These non-standard data types are supported by HabJsonEncoder + json.dumps(NotSet, indent=4, cls=utils.HabJsonEncoder) + json.dumps(datetime.now(), indent=4, cls=utils.HabJsonEncoder) + json.dumps(date.today(), indent=4, cls=utils.HabJsonEncoder) + + # Errors are still raised if passing un-handled data types. + with pytest.raises(TypeError): + json.dumps(object, indent=4, cls=utils.HabJsonEncoder) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 4a073b4..18bd1b4 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -3,13 +3,16 @@ import sys from collections import OrderedDict from pathlib import Path +from zipfile import ZipFile import anytree import pytest from packaging.requirements import Requirement -from hab import NotSet, Resolver, Site, utils +from hab import DistroMode, NotSet, Resolver, Site, utils +from hab.distro_finders.distro_finder import DistroFinder from hab.errors import InvalidRequirementError +from hab.parsers import DistroVersion from hab.solvers import Solver @@ -17,12 +20,14 @@ def test_environment_variables(config_root, helpers, monkeypatch): """Check that Resolver's init respects the environment variables it uses.""" config_paths_env = utils.Platform.expand_paths(["a/config/path", "b/config/path"]) distro_paths_env = utils.Platform.expand_paths(["a/distro/path", "b/distro/path"]) + distro_paths_env = [DistroFinder(p) for p in distro_paths_env] config_paths_direct = utils.Platform.expand_paths( ["z/config/path", "zz/config/path"] ) distro_paths_direct = utils.Platform.expand_paths( ["z/distro/path", "zz/distro/path"] ) + distro_paths_direct = [DistroFinder(p) for p in distro_paths_direct] # Set the config environment variables monkeypatch.setenv( @@ -116,6 +121,52 @@ def test_closest_config(resolver, path, result, reason): assert resolver.closest_config(path).fullpath == result, reason +def test_distro_mode(zip_distro, helpers, tmp_path): + """Test `Resolver.distro_mode` is respected when calling `distros`. + + Also test that the `distro_mode_override` with context updates and restores + the distro_mode. + """ + site_file = tmp_path / "site.json" + helpers.render_template( + "site_download.json", site_file, zip_root=zip_distro.root.as_posix() + ) + resolver = Resolver(Site([site_file])) + + # Install some distros so the resolver can find them + for distro, version in (("dist_a", "0.1"), ("dist_b", "0.5")): + with ZipFile(zip_distro.root / f"{distro}_v{version}.zip") as zip_info: + zip_info.extractall(tmp_path / "distros" / distro / version) + + def get_dist_names(): + return [ + row.node.name + for row in resolver.dump_forest(resolver.distros, attr=None) + if isinstance(row.node, DistroVersion) + ] + + # Get the installed distros and check that `.distros` is the correct return + installed = get_dist_names() + assert resolver.distros is resolver._installed_distros + # Get the download distros and check that `.distros` is the correct return + with resolver.distro_mode_override(DistroMode.Downloaded): + downloads = get_dist_names() + assert resolver.distros is resolver._downloadable_distros + # Check that the installed distros are accessible again + installed_after = get_dist_names() + assert resolver.distros is resolver._installed_distros + + assert installed == ["dist_a==0.1", "dist_b==0.5"] + assert installed_after == installed + assert downloads == [ + "dist_a==0.1", + "dist_a==0.2", + "dist_a==1.0", + "dist_b==0.5", + "dist_b==0.6", + ] + + class TestDumpForest: """Test the dump_forest method on resolver""" @@ -958,19 +1009,19 @@ def test_clear_caches(resolver): """Test that Resolver.clear_cache works as expected.""" # Resolver cache is empty assert resolver._configs is None - assert resolver._distros is None + assert resolver._installed_distros is None # Populate resolver cache data resolver.resolve("not_set") assert isinstance(resolver._configs, dict) - assert isinstance(resolver._distros, dict) + assert isinstance(resolver._installed_distros, dict) assert len(resolver._configs) > 1 - assert len(resolver._distros) > 1 + assert len(resolver._installed_distros) > 1 # Calling clear_caches resets the resolver cache resolver.clear_caches() assert resolver._configs is None - assert resolver._distros is None + assert resolver._installed_distros is None def test_clear_caches_cached(habcached_resolver): diff --git a/tests/test_site.py b/tests/test_site.py index b60d3b8..ee2a83f 100644 --- a/tests/test_site.py +++ b/tests/test_site.py @@ -1,4 +1,5 @@ import sys +import tempfile from pathlib import Path, PurePosixPath, PureWindowsPath import pytest @@ -6,6 +7,7 @@ from hab import Resolver, Site, utils from hab.cache import Cache +from hab.distro_finders.distro_finder import DistroFinder def test_environment_variables(config_root, monkeypatch): @@ -206,7 +208,7 @@ def test_paths(self, config_root, helpers): ) assert len(site.get("distro_paths")) == 2 helpers.check_path_list( - site.get("distro_paths"), + [p.root for p in site.get("distro_paths")], ( config_root / "distros" / "*", config_root / "duplicates" / "distros_1" / "*", @@ -228,7 +230,7 @@ def test_paths_reversed(self, config_root, helpers): ) assert len(site.get("distro_paths")) == 2 helpers.check_path_list( - site.get("distro_paths"), + [p.root for p in site.get("distro_paths")], ( config_root / "duplicates" / "distros_1" / "*", config_root / "distros" / "*", @@ -289,8 +291,8 @@ def test_dump_cached(config_root, habcached_site_file): f" {other_site}", f"{{green}}config_paths: {{reset}}config\\path\\{platform}", f" {config_root}\\configs\\*{{cached}}", - f"{{green}}distro_paths: {{reset}}distro\\path\\{platform}", - f" {config_root}\\distros\\*{{cached}}", + f"{{green}}distro_paths: {{reset}}distro\\path\\{platform}{{cls_name}}", + f" {config_root}\\distros\\*{{cls_name}}{{cached}}", ) check_template = "\n".join(check_template) colors = { @@ -304,13 +306,22 @@ def test_dump_cached(config_root, habcached_site_file): # No verbosity, should not show cached status assert site.get("colorize") is None result = site.dump(width=60) - check = check_template.format(cached="", **colors) + check = check_template.format(cached="", cls_name="", **colors) assert check in result # verbosity enabled, should show cached status result = site.dump(verbosity=1, width=60) check = check_template.format( - cached=f" {Fore.YELLOW}(cached){Style.RESET_ALL}", **colors + cached=f" {Fore.YELLOW}(cached){Style.RESET_ALL}", cls_name="", **colors + ) + assert check in result + + # verbosity level 2, should also show DistroFinder classes + result = site.dump(verbosity=2, width=60) + check = check_template.format( + cached=f" {Fore.YELLOW}(cached){Style.RESET_ALL}", + cls_name=f" {Fore.CYAN}[DistroFinder]{Style.RESET_ALL}", + **colors, ) assert check in result @@ -320,12 +331,19 @@ def test_dump_cached(config_root, habcached_site_file): # No verbosity, should not show cached status result = site.dump(width=60) - check = check_template.format(cached="", green="", reset="") + check = check_template.format(cached="", green="", reset="", cls_name="") assert check in result # verbosity enabled, should show cached status result = site.dump(verbosity=1, width=60) - check = check_template.format(cached=" (cached)", green="", reset="") + check = check_template.format(cached=" (cached)", green="", reset="", cls_name="") + assert check in result + + # verbosity level 2, should also show DistroFinder classes + result = site.dump(verbosity=2, width=60) + check = check_template.format( + cached=" (cached)", green="", reset="", cls_name=" [DistroFinder]" + ) assert check in result @@ -340,7 +358,7 @@ def test_linux(self, monkeypatch, config_root): site = Site(paths) assert site.get("config_paths") == [Path("config/path/linux")] - assert site.get("distro_paths") == [Path("distro/path/linux")] + assert site.get("distro_paths") == [DistroFinder(Path("distro/path/linux"))] assert site.get("platforms") == ["windows", "linux"] def test_osx(self, monkeypatch, config_root): @@ -353,7 +371,7 @@ def test_osx(self, monkeypatch, config_root): site = Site(paths) assert site.get("config_paths") == [Path("config/path/osx")] - assert site.get("distro_paths") == [Path("distro/path/osx")] + assert site.get("distro_paths") == [DistroFinder(Path("distro/path/osx"))] assert site.get("platforms") == ["osx", "linux"] def test_win(self, monkeypatch, config_root): @@ -366,7 +384,7 @@ def test_win(self, monkeypatch, config_root): site = Site(paths) assert site.get("config_paths") == [Path("config\\path\\windows")] - assert site.get("distro_paths") == [Path("distro\\path\\windows")] + assert site.get("distro_paths") == [DistroFinder(Path("distro\\path\\windows"))] assert site.get("platforms") == ["windows", "osx"] @@ -756,3 +774,93 @@ def test_habcache_cls(self, config_root, uncached_resolver): match="hab_test_entry_points.CacheVX class was used", ): Site([config_root / "site" / "eps" / "site_habcache_cls.json"]) + + def test_entry_point_init(self, config_root): + site = Site([config_root / "site_main.json"]) + instance = site.entry_point_init( + "group.name", + "hab.distro_finders.distro_finder:DistroFinder", + ["a/root/path", {"site": "a Site Instance"}], + ) + # The entry_point class was imported and initialized + assert isinstance(instance, DistroFinder) + # The instance had the requested arguments passed to it + assert instance.root == Path("a/root/path") + # The last item was a dictionary, that was removed from args and passed + # as kwargs. + # NOTE: you should not pass site using this method. It's being used here + # to test the kwargs feature and ensure the default site setting doesn't + # overwrite site if it was passed as a kwarg. + assert instance.site == "a Site Instance" + + # Don't pass a kwargs dict, it should get site from itself. + instance = site.entry_point_init( + "group.name", + "hab.distro_finders.distro_finder:DistroFinder", + ["b/root/path"], + ) + assert instance.root == Path("b/root/path") + assert instance.site is site + + +class TestDownloads: + # Defaults to `$TEMP/hab_downloads` if not specified + default_cache_root = Path(tempfile.gettempdir()) / "hab_downloads" + + def test_download_cache(self, config_root, uncached_resolver): + """Test how `site.downloads["cache_root"]` is processed.""" + site = uncached_resolver.site + assert site.downloads["cache_root"] == self.default_cache_root + # `Platform.default_download_cache()` returns the expected default value + assert utils.Platform.default_download_cache() == self.default_cache_root + + # If specified, only the first path is used. This is using a non-valid + # relative path for testing, in practice this should be a absolute path. + paths = [config_root / "site" / "site_distro_finder.json"] + site = Site(paths) + assert ( + site.downloads["cache_root"] == Path("hab testable") / "download" / "path" + ) + + # Use the default if site specifies cache_root but its an empty string. + paths = [config_root / "site" / "site_distro_finder_empty.json"] + site = Site(paths) + assert site.downloads["cache_root"] == self.default_cache_root + + def test_lazy(self, config_root): + site = Site([config_root / "site" / "site_distro_finder.json"]) + # Check that downloads is not parsed before the downloads property + # is first called. + assert site._downloads_parsed is False + downloads = site.downloads + assert site._downloads_parsed is True + assert site.downloads is downloads + + def test_default_settings(self, config_root): + """Test the default downloads values if not defined by site files.""" + site = Site([config_root / "site_main.json"]) + downloads = site.downloads + assert len(downloads["distros"]) == 0 + + # cache_root is always defined + assert downloads["cache_root"] == self.default_cache_root + # These are only defined if the json file defines them. + assert "install_root" not in downloads + assert "relative_path" not in downloads + + def test_all_settings_defined(self, config_root): + """Test the resolved downloads values defined by a site file.""" + from hab.distro_finders.df_zip import DistroFinderZip + + site = Site([config_root / "site" / "site_distro_finder.json"]) + downloads = site.downloads + + # Check that each part of downloads was processed correctly + assert len(downloads["distros"]) == 1 + finder = downloads["distros"][0] + assert isinstance(finder, DistroFinderZip) + assert finder.root == Path("network_server/distro/source") + + assert downloads["cache_root"] == Path("hab testable/download/path") + assert downloads["install_root"] == config_root / "site" / "distros" + assert downloads["relative_path"] == "{distro_name}_v{version}" diff --git a/tox.ini b/tox.ini index 823197b..a738dd3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = begin,py{36,37,38,39,310,311}-{json,json5},end,black,flake8 +envlist = begin,py{37,38,39,310,311}-{json,json5},end,black,flake8 skip_missing_interpreters = True skipsdist = True @@ -29,8 +29,15 @@ commands = coverage erase +[testenv:py{37,38,39,310,311}-{json,json5}] +depends = begin + [testenv:end] basepython = python3 +depends = + begin + py{37,38,39,310,311}-{json,json5} +parallel_show_output = True deps = coverage commands =