diff --git a/tests/conftest.py b/tests/conftest.py index dab4f29..2aed682 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,13 @@ import json import os +import shutil +from collections import namedtuple from contextlib import contextmanager from pathlib import Path, PurePath +from zipfile import ZipFile import pytest +from jinja2 import Environment, FileSystemLoader from packaging.requirements import Requirement from hab import Resolver, Site @@ -111,6 +115,110 @@ def resolver(request): return request.getfixturevalue(test_map[request.param]) +Distro = namedtuple("Distro", ["name", "version", "inc_version", "distros"]) + + +class DistroInfo(namedtuple("DistroInfo", ["root", "versions"])): + default_versions = ( + ("dist_a", "0.1", True, None), + ("dist_a", "0.2", False, ["dist_b"]), + ("dist_a", "1.0", False, None), + ("dist_b", "0.5", False, None), + ("dist_b", "0.6", False, None), + ) + + @classmethod + def dist_version(cls, distro, version): + return f"{distro}_v{version}" + + @classmethod + def hab_json(cls, distro, version=None, distros=None): + data = {"name": distro} + if version: + data["version"] = version + if distros: + data["distros"] = distros + return json.dumps(data, indent=4) + + @classmethod + def generate(cls, root, versions=None, zip_created=None): + if versions is None: + versions = cls.default_versions + + versions = {(x[0], x[1]): Distro(*x) for x in versions} + + for version in versions.values(): + name = cls.dist_version(version.name, version.version) + filename = root / f"{name}.zip" + ver = version.version if version.inc_version else None + with ZipFile(filename, "w") as zf: + zf.writestr( + ".hab.json", + cls.hab_json(version.name, version=ver, distros=version.distros), + ) + zf.writestr("file_a.txt", "File A inside the distro.") + zf.writestr("folder/file_b.txt", "File B inside the distro.") + if zip_created: + zip_created(zf) + + # Create a correctly named .zip file that doesn't have a .hab.json file + # to test for .zip files that are not distros. + with ZipFile(root / "not_valid_v0.1.zip", "w") as zf: + zf.writestr("README.txt", "This file is not a hab distro zip.") + + return cls(root, versions) + + +@pytest.fixture(scope="session") +def distro_finder_info(tmp_path_factory): + """Returns a DistroInfo instance with extracted distros ready for hab. + + This is useful for using an existing hab distro structure as your download server. + """ + root = tmp_path_factory.mktemp("_distro_finder") + + def zip_created(zf): + """Extract all contents zip into a distro folder structure.""" + filename = Path(zf.filename).stem + distro, version = filename.split("_v") + zf.extractall(root / distro / version) + + return DistroInfo.generate(root, zip_created=zip_created) + + +@pytest.fixture(scope="session") +def zip_distro(tmp_path_factory): + """Returns a DistroInfo instance for a zip folder structure. + + This is useful if the zip files are locally accessible or if your hab download + server supports `HTTP range requests`_. For example if you are using Amazon S3. + + .. _HTTP range requests: + https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests + """ + root = tmp_path_factory.mktemp("_zip_distro") + return DistroInfo.generate(root) + + +@pytest.fixture(scope="session") +def zip_distro_sidecar(tmp_path_factory): + """Returns a DistroInfo instance for a zip folder structure with sidecar + `.hab.json` files. + + This is useful when your hab download server does not support HTTP range requests. + """ + root = tmp_path_factory.mktemp("_zip_distro_sidecar") + + def zip_created(zf): + """Extract the .hab.json from the zip to a sidecar file.""" + filename = Path(zf.filename).stem + sidecar = root / f"{filename}.hab.json" + path = zf.extract(".hab.json", root) + shutil.move(path, sidecar) + + return DistroInfo.generate(root, zip_created=zip_created) + + class Helpers(object): """A collection of reusable functions that tests can use.""" @@ -204,6 +312,36 @@ def compare_files(generated, check): cache[i] == check[i] ), f"Difference on line: {i} between the generated cache and {generated}." + @staticmethod + def render_template(template, dest, **kwargs): + """Render a jinja template in from the test templates directory. + + Args: + template (str): The name of the template file in the templates dir. + dest (os.PathLike): The destination filename to write the output. + **kwargs: All kwargs are used to render the template. + """ + environment = Environment( + loader=FileSystemLoader(str(Path(__file__).parent / "templates")), + trim_blocks=True, + lstrip_blocks=True, + ) + template = environment.get_template(template) + + text = template.render(**kwargs).rstrip() + "\n" + with dest.open("w") as fle: + fle.write(text) + + @classmethod + def render_resolver(cls, site_template, dest, **kwargs): + """Calls `render_template` and constructs a Resolver instance for it.""" + # Build the hab site + site_file = dest / "site.json" + cls.render_template(site_template, site_file, **kwargs) + + site = Site([site_file]) + return Resolver(site) + @pytest.fixture def helpers(): diff --git a/tests/site/site_distro_finder.json b/tests/site/site_distro_finder.json new file mode 100644 index 0000000..414caee --- /dev/null +++ b/tests/site/site_distro_finder.json @@ -0,0 +1,32 @@ +{ + "set": + { + "distro_paths": + [ + [ + "hab.distro_finders.distro_finder:DistroFinder", + "hab testable/download/path" + ], + [ + "hab.distro_finders.distro_finder:DistroFinder", + "hab testing/downloads", + { + "site": "for testing only, do not specify site" + } + ] + ], + "downloads": + { + "cache_root": "hab testable/download/path", + "distros": + [ + [ + "hab.distro_finders.df_zip:DistroFinderZip", + "network_server/distro/source" + ] + ], + "install_root": "{relative_root}/distros", + "relative_path": "{{distro_name}}_v{{version}}" + } + } +} diff --git a/tests/site/site_distro_finder_empty.json b/tests/site/site_distro_finder_empty.json new file mode 100644 index 0000000..240abbe --- /dev/null +++ b/tests/site/site_distro_finder_empty.json @@ -0,0 +1,7 @@ +{ + "set": { + "downloads": { + "cache_root": "" + } + } +} diff --git a/tests/templates/site_distro_finder.json b/tests/templates/site_distro_finder.json new file mode 100644 index 0000000..a0b7c7a --- /dev/null +++ b/tests/templates/site_distro_finder.json @@ -0,0 +1,20 @@ +{ + "set": { + "config_paths": [ + "{relative_root}/configs" + ], + "distro_paths": [ + "{relative_root}/distros/*" + ], + "downloads": { + "cache_root": "{relative_root}/downloads", + "distros": [ + [ + "hab.distro_finders.distro_finder:DistroFinder", + "{{ zip_root }}/*" + ] + ], + "install_root": "{relative_root}/distros" + } + } +} diff --git a/tests/templates/site_distro_zip.json b/tests/templates/site_distro_zip.json new file mode 100644 index 0000000..b4fd30e --- /dev/null +++ b/tests/templates/site_distro_zip.json @@ -0,0 +1,20 @@ +{ + "set": { + "config_paths": [ + "{relative_root}/configs" + ], + "distro_paths": [ + "{relative_root}/distros/*" + ], + "downloads": { + "cache_root": "{relative_root}/downloads", + "distros": [ + [ + "hab.distro_finders.df_zip:DistroFinderZip", + "{{ zip_root }}" + ] + ], + "install_root": "{relative_root}/distros" + } + } +} diff --git a/tests/templates/site_distro_zip_sidecar.json b/tests/templates/site_distro_zip_sidecar.json new file mode 100644 index 0000000..689cd83 --- /dev/null +++ b/tests/templates/site_distro_zip_sidecar.json @@ -0,0 +1,20 @@ +{ + "set": { + "config_paths": [ + "{relative_root}/configs" + ], + "distro_paths": [ + "{relative_root}/distros/*" + ], + "downloads": { + "cache_root": "{relative_root}/downloads", + "distros": [ + [ + "hab.distro_finders.zip_sidecar:DistroFinderZipSidecar", + "{{ zip_root }}" + ] + ], + "install_root": "{relative_root}/distros" + } + } +} diff --git a/tests/templates/site_download.json b/tests/templates/site_download.json new file mode 100644 index 0000000..b4fd30e --- /dev/null +++ b/tests/templates/site_download.json @@ -0,0 +1,20 @@ +{ + "set": { + "config_paths": [ + "{relative_root}/configs" + ], + "distro_paths": [ + "{relative_root}/distros/*" + ], + "downloads": { + "cache_root": "{relative_root}/downloads", + "distros": [ + [ + "hab.distro_finders.df_zip:DistroFinderZip", + "{{ zip_root }}" + ] + ], + "install_root": "{relative_root}/distros" + } + } +} diff --git a/tests/test_distro_finder.py b/tests/test_distro_finder.py new file mode 100644 index 0000000..2b8fcc1 --- /dev/null +++ b/tests/test_distro_finder.py @@ -0,0 +1,398 @@ +import glob +import logging +from pathlib import Path + +import pytest + +from hab import DistroMode, Resolver, Site, utils +from hab.distro_finders import df_zip, distro_finder, zip_sidecar +from hab.errors import InstallDestinationExistsError +from hab.parsers import DistroVersion + + +def test_distro_finder_entry_point(config_root): + """Test edge cases for DistroFinder entry_point processing.""" + paths = [config_root / "site" / "site_distro_finder.json"] + site = Site(paths) + distro_paths = site["distro_paths"] + # Ensure the DistroFinder paths are set correctly when set as EntryPoint + assert distro_paths[0].root == Path("hab testable") / "download" / "path" + assert distro_paths[1].root == Path("hab testing") / "downloads" + # The second path passes the kwargs dict with `site`. This triggers testing + # when a dict is passed to the entry_point. However site is always set to + # the current site after a DistroFinder is initialized. + assert distro_paths[1].site == site + + +def test_eq(): + a = distro_finder.DistroFinder("path/a") + + assert a == distro_finder.DistroFinder("path/a") + assert a != distro_finder.DistroFinder("path/b") + + # Test that if the glob_str is different it will not compare equal + b = distro_finder.DistroFinder("path/a") + b.glob_str = "*/test.json" + assert a != b + # Test that if glob_str attr is missing it will not compare equal + del b.glob_str + assert a != b + # Restore glob_str and the objects will compare equal again + b.glob_str = "*/.hab.json" + assert a == b + + # Test that if the root is different it will not compare equal + b.root = Path(".") + assert a != b + # Test that if root attr is missing it will not compare equal + del b.root + assert a != b + # Restore root and the objects will compare equal again + b.root = Path("path/a") + assert a == b + + +@pytest.mark.parametrize( + "glob_str,count", + ( + ("{root}/reference*/sh_*", 12), + ("{root}/reference/*", 0), + ("{root}/reference_scripts/*/*.sh", 20), + ), +) +def test_glob_path(config_root, glob_str, count): + """Ensure `hab.utils.glob_path` returns the expected results.""" + glob_str = glob_str.format(root=config_root) + # Check against the `glob.glob` result. + check = sorted([Path(p) for p in glob.glob(glob_str)]) + + path_with_glob = Path(glob_str) + result = sorted(utils.glob_path(path_with_glob)) + + assert result == check + # Sanity check to ensure that the expected results were found by `glob.glob` + assert len(result) == count + + +class TestLoadPath: + """Test the various `DistroFinder.load_path` implementations.""" + + def test_distro_finder(self, uncached_resolver): + """Currently load_path for DistroFinder just returns None.""" + finder = distro_finder.DistroFinder("", uncached_resolver.site) + assert finder.load_path(Path(".")) is None + + def test_zip_sidecar(self, zip_distro_sidecar): + """The Zip Sidecar reads a .json file next to the zip distro. + + Ensure it's able to read data from the .json file. + """ + finder = zip_sidecar.DistroFinderZipSidecar(zip_distro_sidecar.root) + + # This distro hard codes the version inside the .json file + data = finder.load_path(zip_distro_sidecar.root / "dist_a_v0.1.hab.json") + assert data["name"] == "dist_a" + assert "distros" not in data + assert data["version"] == "0.1" + + # Test a different distro that doesn't hard code the version + data = finder.load_path(zip_distro_sidecar.root / "dist_b_v0.5.hab.json") + assert data["name"] == "dist_b" + assert "distros" not in data + assert data["version"] == "0.5" + + # This distro includes required distros + data = finder.load_path(zip_distro_sidecar.root / "dist_a_v0.2.hab.json") + assert data["name"] == "dist_a" + assert data["distros"] == ["dist_b"] + assert data["version"] == "0.2" + + def test_s3(self): + pass + + +class CheckDistroFinder: + distro_finder_cls = distro_finder.DistroFinder + site_template = "site_distro_finder.json" + + def create_resolver(self, zip_root, helpers, tmp_path): + """Create a hab site for the test.""" + return helpers.render_resolver( + self.site_template, tmp_path, zip_root=zip_root.as_posix() + ) + + def check_installed(self, a_distro_finder, helpers, tmp_path): + resolver = self.create_resolver(a_distro_finder.root, helpers, tmp_path) + finder = resolver.distro_paths[0] + distro_folder = resolver.site.downloads["install_root"] / "dist_a" / "0.1" + + # The distro is not installed yet + assert not distro_folder.exists() + assert not finder.installed(distro_folder) + + # Simulate installing by creating the .hab.json file(contents doesn't matter) + distro_folder.mkdir(parents=True) + with (distro_folder / ".hab.json").open("w"): + pass + assert finder.installed(distro_folder) + + def check_install(self, a_distro_finder, helpers, tmp_path): + resolver = self.create_resolver(a_distro_finder.root, helpers, tmp_path) + dl_finder = resolver.site.downloads["distros"][0] + assert isinstance(dl_finder, self.distro_finder_cls) + install_root = resolver.site.downloads["install_root"] + + for di in a_distro_finder.versions.values(): + # Get the downloadable distro + with resolver.distro_mode_override(DistroMode.Downloaded): + dl_distro = resolver.find_distro(f"{di.name}=={di.version}") + + # Ensure the finder used to create this distro is set + assert dl_distro.finder == dl_finder + + dest = install_root / dl_distro.distro_name / str(dl_distro.version) + assert not dest.exists() + dl_finder.install(dl_distro.filename, dest) + assert dest.is_dir() + assert (dest / ".hab.json").exists() + assert (dest / "file_a.txt").exists() + assert (dest / "folder/file_b.txt").exists() + + # Test that if you try to install an already existing distro + # an exception is raised + with pytest.raises( + InstallDestinationExistsError, match="The destination already exists:" + ) as excinfo: + dl_finder.install(dl_distro.filename, dest) + assert excinfo.value.filename == dest + + +class TestDistroFinder(CheckDistroFinder): + distro_finder_cls = distro_finder.DistroFinder + site_template = "site_distro_finder.json" + + def test_content(self, distro_finder_info): + """Content always returns the parent of the provided path currently.""" + finder = self.distro_finder_cls(distro_finder_info.root) + # We may want to improve this later, but it works for now + path = distro_finder_info.root / ".hab.json" + result = finder.content(path) + assert result == distro_finder_info.root + + def test_installed(self, distro_finder_info, helpers, tmp_path): + self.check_installed(distro_finder_info, helpers, tmp_path) + + def test_install(self, distro_finder_info, helpers, tmp_path): + self.check_install(distro_finder_info, helpers, tmp_path) + + +class TestZipSidecar(CheckDistroFinder): + """Tests specific to `DistroFinderZip`.""" + + distro_finder_cls = zip_sidecar.DistroFinderZipSidecar + site_template = "site_distro_zip_sidecar.json" + + def test_installed(self, zip_distro_sidecar, helpers, tmp_path): + self.check_installed(zip_distro_sidecar, helpers, tmp_path) + + def test_install(self, zip_distro_sidecar, helpers, tmp_path): + self.check_install(zip_distro_sidecar, helpers, tmp_path) + + +class TestZip(CheckDistroFinder): + """Tests specific to `DistroFinderZip`.""" + + distro_finder_cls = df_zip.DistroFinderZip + site_template = "site_distro_zip.json" + + def test_content(self, zip_distro): + finder = df_zip.DistroFinderZip(zip_distro.root) + # If path is already a .zip file, it is just returned + path = zip_distro.root / "already_zip.zip" + result = finder.content(path) + assert result == path + + # The right most .zip file is returned if path has multiple .zip suffixes. + path = zip_distro.root / "a.zip" / "b.zip" + result = finder.content(path) + assert result == path + + # If a member path is passed, return the right most .zip suffix. + member_path = path / ".hab.json" + result = finder.content(member_path) + assert result == path + + # member paths with nested return the right most .zip suffix. + member_path = path / "folder" / "sub-folder" / "file.json" + result = finder.content(member_path) + assert result == path + + # If no .zip suffix is passed, the original path is returned. + path = zip_distro.root / "not_an_archive.txt" + result = finder.content(path) + assert result == path + + def test_load_path(self, zip_distro): + """The Zip finder reads a .json file from inside the zip distro file. + + Ensure it's able to read data from the .json file. + """ + finder = df_zip.DistroFinderZip(zip_distro.root) + + # This distro hard codes the version inside the .json file + data = finder.load_path(zip_distro.root / "dist_a_v0.1.zip") + assert data["name"] == "dist_a" + assert "distros" not in data + assert data["version"] == "0.1" + + # Test a different distro that doesn't hard code the version + data = finder.load_path(zip_distro.root / "dist_b_v0.5.zip") + assert data["name"] == "dist_b" + assert "distros" not in data + assert data["version"] == "0.5" + + # This distro includes required distros + data = finder.load_path(zip_distro.root / "dist_a_v0.2.zip") + assert data["name"] == "dist_a" + assert data["distros"] == ["dist_b"] + assert data["version"] == "0.2" + + def test_zip_get_file_data(self, zip_distro, caplog): + """Test edge cases for `DistroFinderZip.get_file_data`.""" + finder = df_zip.DistroFinderZip(zip_distro.root) + assert finder._cache == {} + + # This file doesn't have a .hab.json file inside it + path = zip_distro.root / "not_valid_v0.1.zip" + data = finder.get_file_data(path) + assert data is None + assert [path / ".hab.json"] == list(finder._cache.keys()) + finder.clear_cache() + + # Check what happens if a member path isn't provided(Just the .zip file path) + path = zip_distro.root / "dist_a_v0.1.zip" + member_path = path / ".hab.json" + caplog.clear() + with caplog.at_level(logging.DEBUG, logger="hab.distro_finders.df_zip"): + data = finder.get_file_data(path) + check = [f'Implicitly added member ".hab.json" to path "{member_path}".'] + assert check == [rec.message for rec in caplog.records] + # The raw data text was read and returned + assert data == b'{\n "name": "dist_a",\n "version": "0.1"\n}' + assert member_path in finder._cache + + # Test that the cache is returned if populated + data = "Data already in the cache" + finder._cache[Path(member_path)] = data + assert finder.get_file_data(member_path) is data + + def test_installed(self, zip_distro, helpers, tmp_path): + self.check_installed(zip_distro, helpers, tmp_path) + + def test_install(self, zip_distro, helpers, tmp_path): + self.check_install(zip_distro, helpers, tmp_path) + + +# TODO: Break this into separate smaller tests of components for each class not this +@pytest.mark.parametrize( + "distro_info", + ( + # "distro_finder_info", + "zip_distro", + "zip_distro_sidecar", + ), +) +def dtest_zip(request, distro_info, helpers, tmp_path): + # Convert the distro_info parameter to testing values. + df_cls = df_zip.DistroFinderZip + hab_json = ".hab.json" + implements_cache = True + parent_type = True + site_filename = "site_distro_zip.json" + if distro_info == "zip_distro_sidecar": + df_cls = zip_sidecar.DistroFinderZipSidecar + hab_json = "{name}_v{ver}.hab.json" + implements_cache = False + parent_type = "sidecar" + site_filename = "site_distro_zip_sidecar.json" + elif distro_info == "distro_finder_info": + df_cls = distro_finder.DistroFinder + implements_cache = False + parent_type = "directory" + site_filename = "site_distro_finder.json" + distro_info = request.getfixturevalue(distro_info) + + site_file = tmp_path / "site.json" + helpers.render_template( + site_filename, site_file, zip_root=distro_info.root.as_posix() + ) + site_distros = tmp_path / "distros" + + check = set([v[:2] for v in distro_info.versions]) + + site = Site([site_file]) + resolver = Resolver(site) + results = set() + # The correct class was resolved + df = resolver.distro_paths[0] + assert type(df) == df_cls + + if implements_cache: + assert df._cache == {} + + for node in resolver.dump_forest(resolver.distros, attr=None): + distro = node.node + if not isinstance(distro, DistroVersion): + continue + + # Ensure the finder used to create this distro is set + assert distro.finder == df + + assert distro.filename.name == hab_json.format( + name=distro.distro_name, ver=distro.version + ) + if parent_type == "zip": + # If the parent is a zip, then the parent is a zip file + assert distro.filename.parent.suffix == ".zip" + assert distro.filename.parent.is_file() + elif parent_type == "sidecar": + # There is a sidecar zip file next to the *.hab.json file + zip_filename = distro.filename.name.replace(".hab.json", ".zip") + assert (distro.filename.parent / zip_filename).is_file() + elif parent_type == "directory": + assert distro.filename.is_file() + assert distro.filename.name == ".hab.json" + + if implements_cache: + assert distro.filename in df._cache + + results.add((distro.distro_name, str(distro.version))) + + # Test the install process extracts all of the files from the zip + dest = site_distros / distro.distro_name / str(distro.version) + assert not dest.exists() + df.install(distro.filename, dest) + assert dest.is_dir() + assert (dest / ".hab.json").exists() + assert (dest / "file_a.txt").exists() + assert (dest / "folder/file_b.txt").exists() + + # Test that if you try to install an already existing distro + # an exception is raised + with pytest.raises( + InstallDestinationExistsError, match="The destination already exists:" + ) as excinfo: + df.install(distro.filename, dest) + assert excinfo.value.filename == dest + + # Test the installed function + # Returns True if passed a distro version folder containing a .hab.json + assert df.installed(dest) + # It returns False if the .hab.json file doesn't exist + assert not df.installed(site_distros) + + if implements_cache: + df.clear_cache() + assert df._cache == {} + + assert results == check diff --git a/tests/test_parsing.py b/tests/test_parsing.py index 4d81378..0be9f17 100644 --- a/tests/test_parsing.py +++ b/tests/test_parsing.py @@ -1,7 +1,9 @@ import copy import json +import pickle import re import sys +from datetime import date, datetime from pathlib import Path import anytree @@ -9,7 +11,8 @@ import setuptools_scm from packaging.version import Version -from hab import NotSet, utils +from hab import NotSet, Resolver, utils +from hab.distro_finders.distro_finder import DistroFinder from hab.errors import ( DuplicateJsonError, HabError, @@ -17,10 +20,10 @@ ReservedVariableNameError, _IgnoredVersionError, ) -from hab.parsers import Config, DistroVersion, FlatConfig +from hab.parsers import Config, DistroVersion, FlatConfig, HabBase -class TestLoadJsonFile: +class TestLoadJson: """Tests various conditions when using `hab.utils.load_json_file` to ensure expected output. """ @@ -34,16 +37,28 @@ def test_missing(self, tmpdir): utils.load_json_file(path) assert Path(excinfo.value.filename) == path + @classmethod + def check_exception(cls, excinfo, native_json, path): + if native_json: + # If built-in json was used, check that filename was appended to the message + assert f'Source("{path}")' in str(excinfo.value) + else: + # If pyjson5 was used, check that the filename was added to str + assert f"'source': {str(path)!r}" in str(excinfo.value) + # Check that the filename was added to the result dict + assert excinfo.value.result["source"] == str(path) + def test_binary(self, tmpdir): """If attempting to read a binary file, filename is included in exception. This is a problem we run into rarely where a text file gets replaced/generated with a binary file containing noting but a lot of null bytes. """ + bin_data = b"\x00" * 32 path = Path(tmpdir) / "binary.json" # Create a binary test file containing multiple binary null values. with path.open("wb") as fle: - fle.write(b"\x00" * 32) + fle.write(bin_data) # Detect if using pyjson5 or not native_json = False @@ -56,15 +71,15 @@ def test_binary(self, tmpdir): else: exc_type = pyjson5.pyjson5.Json5IllegalCharacter + # Test load_json_file with pytest.raises(exc_type) as excinfo: utils.load_json_file(path) + self.check_exception(excinfo, native_json, path) - if native_json: - # If built-in json was used, check that filename was appended to the message - assert f'Filename("{path}")' in str(excinfo.value) - else: - # If pyjson5 was used, check that the filename was added to the result dict - assert f"{{'filename': {str(path)!r}}}" in str(excinfo.value) + # Test loads_json + with pytest.raises(exc_type) as excinfo: + utils.loads_json(bin_data.decode(), path) + self.check_exception(excinfo, native_json, path) def test_config_load(self, uncached_resolver): cfg = Config({}, uncached_resolver) @@ -77,6 +92,18 @@ def test_config_load(self, uncached_resolver): with pytest.raises(FileNotFoundError): cfg.load("invalid_path.json") + def test_loads_json(self, config_root): + """Test that `loads_json` is able to parse a valid json string.""" + filename = config_root / "site_main.json" + with filename.open() as fle: + text = fle.read() + # Test an existing file is able to be parsed successfully. + data = utils.loads_json(text, filename) + # Spot check that we were able to parse data from the file. + assert isinstance(data, dict) + assert "append" in data + assert "set" in data + def test_distro_parse(config_root, resolver): """Check that a distro json can be parsed correctly""" @@ -182,13 +209,18 @@ def get_ver(*args, **kwargs): app.load(path) -def test_distro_version(resolver): +def test_distro_version(resolver, zip_distro_sidecar): """Verify that we find the expected version for a given requirement.""" maya = resolver.distros["maya2020"] assert maya.latest_version("maya2020").name == "maya2020==2020.1" assert maya.latest_version("maya2020<2020.1").name == "maya2020==2020.0" + forest = {} + resolver = Resolver() + parsed = HabBase(forest, resolver, zip_distro_sidecar.root / "dist_a_v0.1.hab.json") + assert parsed.version == Version("0.1") + def test_config_parse(config_root, resolver, helpers): """Check that a config json can be parsed correctly""" @@ -291,6 +323,7 @@ def test_metaclass(): "environment", "environment_config", "filename", + "finder", "min_verbosity", "name", "optional_distros", @@ -686,19 +719,16 @@ def test_invalid_config(config_root, resolver): with pytest.raises(_JsonException) as excinfo: Config({}, resolver, filename=path) - - if native_json: - # If built-in json was used, check that filename was appended to the message - assert f'Filename("{path}")' in str(excinfo.value) - else: - # If pyjson5 was used, check that the filename was added to the result dict - assert excinfo.value.result["filename"] == str(path) + TestLoadJson.check_exception(excinfo, native_json, path) def test_misc_coverage(resolver): """Test that cover misc lines not covered by the rest of the tests""" assert str(NotSet) == "NotSet" assert copy.copy(NotSet) is NotSet + # Check that NotSet can be pickled + payload = pickle.dumps(NotSet) + assert pickle.loads(payload) is NotSet # Check that dirname is modified when setting a blank filename cfg = Config({}, resolver) @@ -761,10 +791,13 @@ def test_duplicated_distros(config_root, resolver): definitions are in the same config_path so a DuplicateJsonError is raised. """ original = resolver.distro_paths + site = resolver.site # Check that the first config in distro_paths was used distro_paths = list(original) - distro_paths.insert(0, config_root / "duplicates" / "distros_1" / "*") + distro_paths.insert( + 0, DistroFinder(config_root / "duplicates" / "distros_1" / "*", site=site) + ) resolver.distro_paths = distro_paths dcc = resolver.find_distro("the_dcc==1.2") @@ -774,7 +807,9 @@ def test_duplicated_distros(config_root, resolver): # Check that an exception is raised if there are duplicate definitions from # the same distro_paths directory. distro_paths = list(original) - distro_paths.insert(0, config_root / "duplicates" / "distros_2" / "*") + distro_paths.insert( + 0, DistroFinder(config_root / "duplicates" / "distros_2" / "*", site=site) + ) resolver.distro_paths = distro_paths with pytest.raises(DuplicateJsonError): @@ -1172,7 +1207,7 @@ def test_reserved(self, uncached_resolver, variables, invalid, tmpdir): # Add the test distro to hab's distro search. We don't need to call # `clear_caches` because distros haven't been resolved yet. - uncached_resolver.distro_paths.append(Path(tmpdir)) + uncached_resolver.distro_paths.append(DistroFinder(Path(tmpdir))) # When distros are resolved, an exception should be raised with pytest.raises( @@ -1180,3 +1215,14 @@ def test_reserved(self, uncached_resolver, variables, invalid, tmpdir): match=rf"'{invalid}' are reserved variable name\(s\) for hab", ): uncached_resolver.distros + + +def test_hab_json_encoder(): + # These non-standard data types are supported by HabJsonEncoder + json.dumps(NotSet, indent=4, cls=utils.HabJsonEncoder) + json.dumps(datetime.now(), indent=4, cls=utils.HabJsonEncoder) + json.dumps(date.today(), indent=4, cls=utils.HabJsonEncoder) + + # Errors are still raised if passing un-handled data types. + with pytest.raises(TypeError): + json.dumps(object, indent=4, cls=utils.HabJsonEncoder) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 4a073b4..18bd1b4 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -3,13 +3,16 @@ import sys from collections import OrderedDict from pathlib import Path +from zipfile import ZipFile import anytree import pytest from packaging.requirements import Requirement -from hab import NotSet, Resolver, Site, utils +from hab import DistroMode, NotSet, Resolver, Site, utils +from hab.distro_finders.distro_finder import DistroFinder from hab.errors import InvalidRequirementError +from hab.parsers import DistroVersion from hab.solvers import Solver @@ -17,12 +20,14 @@ def test_environment_variables(config_root, helpers, monkeypatch): """Check that Resolver's init respects the environment variables it uses.""" config_paths_env = utils.Platform.expand_paths(["a/config/path", "b/config/path"]) distro_paths_env = utils.Platform.expand_paths(["a/distro/path", "b/distro/path"]) + distro_paths_env = [DistroFinder(p) for p in distro_paths_env] config_paths_direct = utils.Platform.expand_paths( ["z/config/path", "zz/config/path"] ) distro_paths_direct = utils.Platform.expand_paths( ["z/distro/path", "zz/distro/path"] ) + distro_paths_direct = [DistroFinder(p) for p in distro_paths_direct] # Set the config environment variables monkeypatch.setenv( @@ -116,6 +121,52 @@ def test_closest_config(resolver, path, result, reason): assert resolver.closest_config(path).fullpath == result, reason +def test_distro_mode(zip_distro, helpers, tmp_path): + """Test `Resolver.distro_mode` is respected when calling `distros`. + + Also test that the `distro_mode_override` with context updates and restores + the distro_mode. + """ + site_file = tmp_path / "site.json" + helpers.render_template( + "site_download.json", site_file, zip_root=zip_distro.root.as_posix() + ) + resolver = Resolver(Site([site_file])) + + # Install some distros so the resolver can find them + for distro, version in (("dist_a", "0.1"), ("dist_b", "0.5")): + with ZipFile(zip_distro.root / f"{distro}_v{version}.zip") as zip_info: + zip_info.extractall(tmp_path / "distros" / distro / version) + + def get_dist_names(): + return [ + row.node.name + for row in resolver.dump_forest(resolver.distros, attr=None) + if isinstance(row.node, DistroVersion) + ] + + # Get the installed distros and check that `.distros` is the correct return + installed = get_dist_names() + assert resolver.distros is resolver._installed_distros + # Get the download distros and check that `.distros` is the correct return + with resolver.distro_mode_override(DistroMode.Downloaded): + downloads = get_dist_names() + assert resolver.distros is resolver._downloadable_distros + # Check that the installed distros are accessible again + installed_after = get_dist_names() + assert resolver.distros is resolver._installed_distros + + assert installed == ["dist_a==0.1", "dist_b==0.5"] + assert installed_after == installed + assert downloads == [ + "dist_a==0.1", + "dist_a==0.2", + "dist_a==1.0", + "dist_b==0.5", + "dist_b==0.6", + ] + + class TestDumpForest: """Test the dump_forest method on resolver""" @@ -958,19 +1009,19 @@ def test_clear_caches(resolver): """Test that Resolver.clear_cache works as expected.""" # Resolver cache is empty assert resolver._configs is None - assert resolver._distros is None + assert resolver._installed_distros is None # Populate resolver cache data resolver.resolve("not_set") assert isinstance(resolver._configs, dict) - assert isinstance(resolver._distros, dict) + assert isinstance(resolver._installed_distros, dict) assert len(resolver._configs) > 1 - assert len(resolver._distros) > 1 + assert len(resolver._installed_distros) > 1 # Calling clear_caches resets the resolver cache resolver.clear_caches() assert resolver._configs is None - assert resolver._distros is None + assert resolver._installed_distros is None def test_clear_caches_cached(habcached_resolver): diff --git a/tests/test_site.py b/tests/test_site.py index b60d3b8..ee2a83f 100644 --- a/tests/test_site.py +++ b/tests/test_site.py @@ -1,4 +1,5 @@ import sys +import tempfile from pathlib import Path, PurePosixPath, PureWindowsPath import pytest @@ -6,6 +7,7 @@ from hab import Resolver, Site, utils from hab.cache import Cache +from hab.distro_finders.distro_finder import DistroFinder def test_environment_variables(config_root, monkeypatch): @@ -206,7 +208,7 @@ def test_paths(self, config_root, helpers): ) assert len(site.get("distro_paths")) == 2 helpers.check_path_list( - site.get("distro_paths"), + [p.root for p in site.get("distro_paths")], ( config_root / "distros" / "*", config_root / "duplicates" / "distros_1" / "*", @@ -228,7 +230,7 @@ def test_paths_reversed(self, config_root, helpers): ) assert len(site.get("distro_paths")) == 2 helpers.check_path_list( - site.get("distro_paths"), + [p.root for p in site.get("distro_paths")], ( config_root / "duplicates" / "distros_1" / "*", config_root / "distros" / "*", @@ -289,8 +291,8 @@ def test_dump_cached(config_root, habcached_site_file): f" {other_site}", f"{{green}}config_paths: {{reset}}config\\path\\{platform}", f" {config_root}\\configs\\*{{cached}}", - f"{{green}}distro_paths: {{reset}}distro\\path\\{platform}", - f" {config_root}\\distros\\*{{cached}}", + f"{{green}}distro_paths: {{reset}}distro\\path\\{platform}{{cls_name}}", + f" {config_root}\\distros\\*{{cls_name}}{{cached}}", ) check_template = "\n".join(check_template) colors = { @@ -304,13 +306,22 @@ def test_dump_cached(config_root, habcached_site_file): # No verbosity, should not show cached status assert site.get("colorize") is None result = site.dump(width=60) - check = check_template.format(cached="", **colors) + check = check_template.format(cached="", cls_name="", **colors) assert check in result # verbosity enabled, should show cached status result = site.dump(verbosity=1, width=60) check = check_template.format( - cached=f" {Fore.YELLOW}(cached){Style.RESET_ALL}", **colors + cached=f" {Fore.YELLOW}(cached){Style.RESET_ALL}", cls_name="", **colors + ) + assert check in result + + # verbosity level 2, should also show DistroFinder classes + result = site.dump(verbosity=2, width=60) + check = check_template.format( + cached=f" {Fore.YELLOW}(cached){Style.RESET_ALL}", + cls_name=f" {Fore.CYAN}[DistroFinder]{Style.RESET_ALL}", + **colors, ) assert check in result @@ -320,12 +331,19 @@ def test_dump_cached(config_root, habcached_site_file): # No verbosity, should not show cached status result = site.dump(width=60) - check = check_template.format(cached="", green="", reset="") + check = check_template.format(cached="", green="", reset="", cls_name="") assert check in result # verbosity enabled, should show cached status result = site.dump(verbosity=1, width=60) - check = check_template.format(cached=" (cached)", green="", reset="") + check = check_template.format(cached=" (cached)", green="", reset="", cls_name="") + assert check in result + + # verbosity level 2, should also show DistroFinder classes + result = site.dump(verbosity=2, width=60) + check = check_template.format( + cached=" (cached)", green="", reset="", cls_name=" [DistroFinder]" + ) assert check in result @@ -340,7 +358,7 @@ def test_linux(self, monkeypatch, config_root): site = Site(paths) assert site.get("config_paths") == [Path("config/path/linux")] - assert site.get("distro_paths") == [Path("distro/path/linux")] + assert site.get("distro_paths") == [DistroFinder(Path("distro/path/linux"))] assert site.get("platforms") == ["windows", "linux"] def test_osx(self, monkeypatch, config_root): @@ -353,7 +371,7 @@ def test_osx(self, monkeypatch, config_root): site = Site(paths) assert site.get("config_paths") == [Path("config/path/osx")] - assert site.get("distro_paths") == [Path("distro/path/osx")] + assert site.get("distro_paths") == [DistroFinder(Path("distro/path/osx"))] assert site.get("platforms") == ["osx", "linux"] def test_win(self, monkeypatch, config_root): @@ -366,7 +384,7 @@ def test_win(self, monkeypatch, config_root): site = Site(paths) assert site.get("config_paths") == [Path("config\\path\\windows")] - assert site.get("distro_paths") == [Path("distro\\path\\windows")] + assert site.get("distro_paths") == [DistroFinder(Path("distro\\path\\windows"))] assert site.get("platforms") == ["windows", "osx"] @@ -756,3 +774,93 @@ def test_habcache_cls(self, config_root, uncached_resolver): match="hab_test_entry_points.CacheVX class was used", ): Site([config_root / "site" / "eps" / "site_habcache_cls.json"]) + + def test_entry_point_init(self, config_root): + site = Site([config_root / "site_main.json"]) + instance = site.entry_point_init( + "group.name", + "hab.distro_finders.distro_finder:DistroFinder", + ["a/root/path", {"site": "a Site Instance"}], + ) + # The entry_point class was imported and initialized + assert isinstance(instance, DistroFinder) + # The instance had the requested arguments passed to it + assert instance.root == Path("a/root/path") + # The last item was a dictionary, that was removed from args and passed + # as kwargs. + # NOTE: you should not pass site using this method. It's being used here + # to test the kwargs feature and ensure the default site setting doesn't + # overwrite site if it was passed as a kwarg. + assert instance.site == "a Site Instance" + + # Don't pass a kwargs dict, it should get site from itself. + instance = site.entry_point_init( + "group.name", + "hab.distro_finders.distro_finder:DistroFinder", + ["b/root/path"], + ) + assert instance.root == Path("b/root/path") + assert instance.site is site + + +class TestDownloads: + # Defaults to `$TEMP/hab_downloads` if not specified + default_cache_root = Path(tempfile.gettempdir()) / "hab_downloads" + + def test_download_cache(self, config_root, uncached_resolver): + """Test how `site.downloads["cache_root"]` is processed.""" + site = uncached_resolver.site + assert site.downloads["cache_root"] == self.default_cache_root + # `Platform.default_download_cache()` returns the expected default value + assert utils.Platform.default_download_cache() == self.default_cache_root + + # If specified, only the first path is used. This is using a non-valid + # relative path for testing, in practice this should be a absolute path. + paths = [config_root / "site" / "site_distro_finder.json"] + site = Site(paths) + assert ( + site.downloads["cache_root"] == Path("hab testable") / "download" / "path" + ) + + # Use the default if site specifies cache_root but its an empty string. + paths = [config_root / "site" / "site_distro_finder_empty.json"] + site = Site(paths) + assert site.downloads["cache_root"] == self.default_cache_root + + def test_lazy(self, config_root): + site = Site([config_root / "site" / "site_distro_finder.json"]) + # Check that downloads is not parsed before the downloads property + # is first called. + assert site._downloads_parsed is False + downloads = site.downloads + assert site._downloads_parsed is True + assert site.downloads is downloads + + def test_default_settings(self, config_root): + """Test the default downloads values if not defined by site files.""" + site = Site([config_root / "site_main.json"]) + downloads = site.downloads + assert len(downloads["distros"]) == 0 + + # cache_root is always defined + assert downloads["cache_root"] == self.default_cache_root + # These are only defined if the json file defines them. + assert "install_root" not in downloads + assert "relative_path" not in downloads + + def test_all_settings_defined(self, config_root): + """Test the resolved downloads values defined by a site file.""" + from hab.distro_finders.df_zip import DistroFinderZip + + site = Site([config_root / "site" / "site_distro_finder.json"]) + downloads = site.downloads + + # Check that each part of downloads was processed correctly + assert len(downloads["distros"]) == 1 + finder = downloads["distros"][0] + assert isinstance(finder, DistroFinderZip) + assert finder.root == Path("network_server/distro/source") + + assert downloads["cache_root"] == Path("hab testable/download/path") + assert downloads["install_root"] == config_root / "site" / "distros" + assert downloads["relative_path"] == "{distro_name}_v{version}"