From 5570f9a550a536702c20872c9073c4fe54a756e4 Mon Sep 17 00:00:00 2001
From: Mike Hendricks <mikeh@blur.com>
Date: Fri, 6 Dec 2024 12:28:51 -0800
Subject: [PATCH] 4/4: Add testing for the new hab install and DistroFinder
 features

---
 tests/conftest.py                            | 138 +++++++
 tests/site/site_distro_finder.json           |  32 ++
 tests/site/site_distro_finder_empty.json     |   7 +
 tests/templates/site_distro_finder.json      |  20 +
 tests/templates/site_distro_zip.json         |  20 +
 tests/templates/site_distro_zip_sidecar.json |  20 +
 tests/templates/site_download.json           |  20 +
 tests/test_distro_finder.py                  | 398 +++++++++++++++++++
 tests/test_parsing.py                        |  88 +++-
 tests/test_resolver.py                       |  61 ++-
 tests/test_site.py                           | 130 +++++-
 11 files changed, 897 insertions(+), 37 deletions(-)
 create mode 100644 tests/site/site_distro_finder.json
 create mode 100644 tests/site/site_distro_finder_empty.json
 create mode 100644 tests/templates/site_distro_finder.json
 create mode 100644 tests/templates/site_distro_zip.json
 create mode 100644 tests/templates/site_distro_zip_sidecar.json
 create mode 100644 tests/templates/site_download.json
 create mode 100644 tests/test_distro_finder.py

diff --git a/tests/conftest.py b/tests/conftest.py
index dab4f29..2aed682 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,9 +1,13 @@
 import json
 import os
+import shutil
+from collections import namedtuple
 from contextlib import contextmanager
 from pathlib import Path, PurePath
+from zipfile import ZipFile
 
 import pytest
+from jinja2 import Environment, FileSystemLoader
 from packaging.requirements import Requirement
 
 from hab import Resolver, Site
@@ -111,6 +115,110 @@ def resolver(request):
     return request.getfixturevalue(test_map[request.param])
 
 
+Distro = namedtuple("Distro", ["name", "version", "inc_version", "distros"])
+
+
+class DistroInfo(namedtuple("DistroInfo", ["root", "versions"])):
+    default_versions = (
+        ("dist_a", "0.1", True, None),
+        ("dist_a", "0.2", False, ["dist_b"]),
+        ("dist_a", "1.0", False, None),
+        ("dist_b", "0.5", False, None),
+        ("dist_b", "0.6", False, None),
+    )
+
+    @classmethod
+    def dist_version(cls, distro, version):
+        return f"{distro}_v{version}"
+
+    @classmethod
+    def hab_json(cls, distro, version=None, distros=None):
+        data = {"name": distro}
+        if version:
+            data["version"] = version
+        if distros:
+            data["distros"] = distros
+        return json.dumps(data, indent=4)
+
+    @classmethod
+    def generate(cls, root, versions=None, zip_created=None):
+        if versions is None:
+            versions = cls.default_versions
+
+        versions = {(x[0], x[1]): Distro(*x) for x in versions}
+
+        for version in versions.values():
+            name = cls.dist_version(version.name, version.version)
+            filename = root / f"{name}.zip"
+            ver = version.version if version.inc_version else None
+            with ZipFile(filename, "w") as zf:
+                zf.writestr(
+                    ".hab.json",
+                    cls.hab_json(version.name, version=ver, distros=version.distros),
+                )
+                zf.writestr("file_a.txt", "File A inside the distro.")
+                zf.writestr("folder/file_b.txt", "File B inside the distro.")
+                if zip_created:
+                    zip_created(zf)
+
+        # Create a correctly named .zip file that doesn't have a .hab.json file
+        # to test for .zip files that are not distros.
+        with ZipFile(root / "not_valid_v0.1.zip", "w") as zf:
+            zf.writestr("README.txt", "This file is not a hab distro zip.")
+
+        return cls(root, versions)
+
+
+@pytest.fixture(scope="session")
+def distro_finder_info(tmp_path_factory):
+    """Returns a DistroInfo instance with extracted distros ready for hab.
+
+    This is useful for using an existing hab distro structure as your download server.
+    """
+    root = tmp_path_factory.mktemp("_distro_finder")
+
+    def zip_created(zf):
+        """Extract all contents zip into a distro folder structure."""
+        filename = Path(zf.filename).stem
+        distro, version = filename.split("_v")
+        zf.extractall(root / distro / version)
+
+    return DistroInfo.generate(root, zip_created=zip_created)
+
+
+@pytest.fixture(scope="session")
+def zip_distro(tmp_path_factory):
+    """Returns a DistroInfo instance for a zip folder structure.
+
+    This is useful if the zip files are locally accessible or if your hab download
+    server supports `HTTP range requests`_. For example if you are using Amazon S3.
+
+    .. _HTTP range requests:
+       https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests
+    """
+    root = tmp_path_factory.mktemp("_zip_distro")
+    return DistroInfo.generate(root)
+
+
+@pytest.fixture(scope="session")
+def zip_distro_sidecar(tmp_path_factory):
+    """Returns a DistroInfo instance for a zip folder structure with sidecar
+    `.hab.json` files.
+
+    This is useful when your hab download server does not support HTTP range requests.
+    """
+    root = tmp_path_factory.mktemp("_zip_distro_sidecar")
+
+    def zip_created(zf):
+        """Extract the .hab.json from the zip to a sidecar file."""
+        filename = Path(zf.filename).stem
+        sidecar = root / f"{filename}.hab.json"
+        path = zf.extract(".hab.json", root)
+        shutil.move(path, sidecar)
+
+    return DistroInfo.generate(root, zip_created=zip_created)
+
+
 class Helpers(object):
     """A collection of reusable functions that tests can use."""
 
@@ -204,6 +312,36 @@ def compare_files(generated, check):
                 cache[i] == check[i]
             ), f"Difference on line: {i} between the generated cache and {generated}."
 
+    @staticmethod
+    def render_template(template, dest, **kwargs):
+        """Render a jinja template in from the test templates directory.
+
+        Args:
+            template (str): The name of the template file in the templates dir.
+            dest (os.PathLike): The destination filename to write the output.
+            **kwargs: All kwargs are used to render the template.
+        """
+        environment = Environment(
+            loader=FileSystemLoader(str(Path(__file__).parent / "templates")),
+            trim_blocks=True,
+            lstrip_blocks=True,
+        )
+        template = environment.get_template(template)
+
+        text = template.render(**kwargs).rstrip() + "\n"
+        with dest.open("w") as fle:
+            fle.write(text)
+
+    @classmethod
+    def render_resolver(cls, site_template, dest, **kwargs):
+        """Calls `render_template` and constructs a Resolver instance for it."""
+        # Build the hab site
+        site_file = dest / "site.json"
+        cls.render_template(site_template, site_file, **kwargs)
+
+        site = Site([site_file])
+        return Resolver(site)
+
 
 @pytest.fixture
 def helpers():
diff --git a/tests/site/site_distro_finder.json b/tests/site/site_distro_finder.json
new file mode 100644
index 0000000..414caee
--- /dev/null
+++ b/tests/site/site_distro_finder.json
@@ -0,0 +1,32 @@
+{
+    "set":
+    {
+        "distro_paths":
+        [
+            [
+                "hab.distro_finders.distro_finder:DistroFinder",
+                "hab testable/download/path"
+            ],
+            [
+                "hab.distro_finders.distro_finder:DistroFinder",
+                "hab testing/downloads",
+                {
+                    "site": "for testing only, do not specify site"
+                }
+            ]
+        ],
+        "downloads":
+        {
+            "cache_root": "hab testable/download/path",
+            "distros":
+            [
+                [
+                    "hab.distro_finders.df_zip:DistroFinderZip",
+                    "network_server/distro/source"
+                ]
+            ],
+            "install_root": "{relative_root}/distros",
+            "relative_path": "{{distro_name}}_v{{version}}"
+        }
+    }
+}
diff --git a/tests/site/site_distro_finder_empty.json b/tests/site/site_distro_finder_empty.json
new file mode 100644
index 0000000..240abbe
--- /dev/null
+++ b/tests/site/site_distro_finder_empty.json
@@ -0,0 +1,7 @@
+{
+    "set": {
+        "downloads": {
+            "cache_root": ""
+        }
+    }
+}
diff --git a/tests/templates/site_distro_finder.json b/tests/templates/site_distro_finder.json
new file mode 100644
index 0000000..a0b7c7a
--- /dev/null
+++ b/tests/templates/site_distro_finder.json
@@ -0,0 +1,20 @@
+{
+    "set": {
+        "config_paths": [
+            "{relative_root}/configs"
+        ],
+        "distro_paths": [
+            "{relative_root}/distros/*"
+        ],
+        "downloads": {
+            "cache_root": "{relative_root}/downloads",
+            "distros": [
+                [
+                    "hab.distro_finders.distro_finder:DistroFinder",
+                    "{{ zip_root }}/*"
+                ]
+            ],
+            "install_root": "{relative_root}/distros"
+        }
+    }
+}
diff --git a/tests/templates/site_distro_zip.json b/tests/templates/site_distro_zip.json
new file mode 100644
index 0000000..b4fd30e
--- /dev/null
+++ b/tests/templates/site_distro_zip.json
@@ -0,0 +1,20 @@
+{
+    "set": {
+        "config_paths": [
+            "{relative_root}/configs"
+        ],
+        "distro_paths": [
+            "{relative_root}/distros/*"
+        ],
+        "downloads": {
+            "cache_root": "{relative_root}/downloads",
+            "distros": [
+                [
+                    "hab.distro_finders.df_zip:DistroFinderZip",
+                    "{{ zip_root }}"
+                ]
+            ],
+            "install_root": "{relative_root}/distros"
+        }
+    }
+}
diff --git a/tests/templates/site_distro_zip_sidecar.json b/tests/templates/site_distro_zip_sidecar.json
new file mode 100644
index 0000000..689cd83
--- /dev/null
+++ b/tests/templates/site_distro_zip_sidecar.json
@@ -0,0 +1,20 @@
+{
+    "set": {
+        "config_paths": [
+            "{relative_root}/configs"
+        ],
+        "distro_paths": [
+            "{relative_root}/distros/*"
+        ],
+        "downloads": {
+            "cache_root": "{relative_root}/downloads",
+            "distros": [
+                [
+                    "hab.distro_finders.zip_sidecar:DistroFinderZipSidecar",
+                    "{{ zip_root }}"
+                ]
+            ],
+            "install_root": "{relative_root}/distros"
+        }
+    }
+}
diff --git a/tests/templates/site_download.json b/tests/templates/site_download.json
new file mode 100644
index 0000000..b4fd30e
--- /dev/null
+++ b/tests/templates/site_download.json
@@ -0,0 +1,20 @@
+{
+    "set": {
+        "config_paths": [
+            "{relative_root}/configs"
+        ],
+        "distro_paths": [
+            "{relative_root}/distros/*"
+        ],
+        "downloads": {
+            "cache_root": "{relative_root}/downloads",
+            "distros": [
+                [
+                    "hab.distro_finders.df_zip:DistroFinderZip",
+                    "{{ zip_root }}"
+                ]
+            ],
+            "install_root": "{relative_root}/distros"
+        }
+    }
+}
diff --git a/tests/test_distro_finder.py b/tests/test_distro_finder.py
new file mode 100644
index 0000000..2b8fcc1
--- /dev/null
+++ b/tests/test_distro_finder.py
@@ -0,0 +1,398 @@
+import glob
+import logging
+from pathlib import Path
+
+import pytest
+
+from hab import DistroMode, Resolver, Site, utils
+from hab.distro_finders import df_zip, distro_finder, zip_sidecar
+from hab.errors import InstallDestinationExistsError
+from hab.parsers import DistroVersion
+
+
+def test_distro_finder_entry_point(config_root):
+    """Test edge cases for DistroFinder entry_point processing."""
+    paths = [config_root / "site" / "site_distro_finder.json"]
+    site = Site(paths)
+    distro_paths = site["distro_paths"]
+    # Ensure the DistroFinder paths are set correctly when set as EntryPoint
+    assert distro_paths[0].root == Path("hab testable") / "download" / "path"
+    assert distro_paths[1].root == Path("hab testing") / "downloads"
+    # The second path passes the kwargs dict with `site`. This triggers testing
+    # when a dict is passed to the entry_point. However site is always set to
+    # the current site after a DistroFinder is initialized.
+    assert distro_paths[1].site == site
+
+
+def test_eq():
+    a = distro_finder.DistroFinder("path/a")
+
+    assert a == distro_finder.DistroFinder("path/a")
+    assert a != distro_finder.DistroFinder("path/b")
+
+    # Test that if the glob_str is different it will not compare equal
+    b = distro_finder.DistroFinder("path/a")
+    b.glob_str = "*/test.json"
+    assert a != b
+    # Test that if glob_str attr is missing it will not compare equal
+    del b.glob_str
+    assert a != b
+    # Restore glob_str and the objects will compare equal again
+    b.glob_str = "*/.hab.json"
+    assert a == b
+
+    # Test that if the root is different it will not compare equal
+    b.root = Path(".")
+    assert a != b
+    # Test that if root attr is missing it will not compare equal
+    del b.root
+    assert a != b
+    # Restore root and the objects will compare equal again
+    b.root = Path("path/a")
+    assert a == b
+
+
+@pytest.mark.parametrize(
+    "glob_str,count",
+    (
+        ("{root}/reference*/sh_*", 12),
+        ("{root}/reference/*", 0),
+        ("{root}/reference_scripts/*/*.sh", 20),
+    ),
+)
+def test_glob_path(config_root, glob_str, count):
+    """Ensure `hab.utils.glob_path` returns the expected results."""
+    glob_str = glob_str.format(root=config_root)
+    # Check against the `glob.glob` result.
+    check = sorted([Path(p) for p in glob.glob(glob_str)])
+
+    path_with_glob = Path(glob_str)
+    result = sorted(utils.glob_path(path_with_glob))
+
+    assert result == check
+    # Sanity check to ensure that the expected results were found by `glob.glob`
+    assert len(result) == count
+
+
+class TestLoadPath:
+    """Test the various `DistroFinder.load_path` implementations."""
+
+    def test_distro_finder(self, uncached_resolver):
+        """Currently load_path for DistroFinder just returns None."""
+        finder = distro_finder.DistroFinder("", uncached_resolver.site)
+        assert finder.load_path(Path(".")) is None
+
+    def test_zip_sidecar(self, zip_distro_sidecar):
+        """The Zip Sidecar reads a .json file next to the zip distro.
+
+        Ensure it's able to read data from the .json file.
+        """
+        finder = zip_sidecar.DistroFinderZipSidecar(zip_distro_sidecar.root)
+
+        # This distro hard codes the version inside the .json file
+        data = finder.load_path(zip_distro_sidecar.root / "dist_a_v0.1.hab.json")
+        assert data["name"] == "dist_a"
+        assert "distros" not in data
+        assert data["version"] == "0.1"
+
+        # Test a different distro that doesn't hard code the version
+        data = finder.load_path(zip_distro_sidecar.root / "dist_b_v0.5.hab.json")
+        assert data["name"] == "dist_b"
+        assert "distros" not in data
+        assert data["version"] == "0.5"
+
+        # This distro includes required distros
+        data = finder.load_path(zip_distro_sidecar.root / "dist_a_v0.2.hab.json")
+        assert data["name"] == "dist_a"
+        assert data["distros"] == ["dist_b"]
+        assert data["version"] == "0.2"
+
+    def test_s3(self):
+        pass
+
+
+class CheckDistroFinder:
+    distro_finder_cls = distro_finder.DistroFinder
+    site_template = "site_distro_finder.json"
+
+    def create_resolver(self, zip_root, helpers, tmp_path):
+        """Create a hab site for the test."""
+        return helpers.render_resolver(
+            self.site_template, tmp_path, zip_root=zip_root.as_posix()
+        )
+
+    def check_installed(self, a_distro_finder, helpers, tmp_path):
+        resolver = self.create_resolver(a_distro_finder.root, helpers, tmp_path)
+        finder = resolver.distro_paths[0]
+        distro_folder = resolver.site.downloads["install_root"] / "dist_a" / "0.1"
+
+        # The distro is not installed yet
+        assert not distro_folder.exists()
+        assert not finder.installed(distro_folder)
+
+        # Simulate installing by creating the .hab.json file(contents doesn't matter)
+        distro_folder.mkdir(parents=True)
+        with (distro_folder / ".hab.json").open("w"):
+            pass
+        assert finder.installed(distro_folder)
+
+    def check_install(self, a_distro_finder, helpers, tmp_path):
+        resolver = self.create_resolver(a_distro_finder.root, helpers, tmp_path)
+        dl_finder = resolver.site.downloads["distros"][0]
+        assert isinstance(dl_finder, self.distro_finder_cls)
+        install_root = resolver.site.downloads["install_root"]
+
+        for di in a_distro_finder.versions.values():
+            # Get the downloadable distro
+            with resolver.distro_mode_override(DistroMode.Downloaded):
+                dl_distro = resolver.find_distro(f"{di.name}=={di.version}")
+
+            # Ensure the finder used to create this distro is set
+            assert dl_distro.finder == dl_finder
+
+            dest = install_root / dl_distro.distro_name / str(dl_distro.version)
+            assert not dest.exists()
+            dl_finder.install(dl_distro.filename, dest)
+            assert dest.is_dir()
+            assert (dest / ".hab.json").exists()
+            assert (dest / "file_a.txt").exists()
+            assert (dest / "folder/file_b.txt").exists()
+
+            # Test that if you try to install an already existing distro
+            # an exception is raised
+            with pytest.raises(
+                InstallDestinationExistsError, match="The destination already exists:"
+            ) as excinfo:
+                dl_finder.install(dl_distro.filename, dest)
+            assert excinfo.value.filename == dest
+
+
+class TestDistroFinder(CheckDistroFinder):
+    distro_finder_cls = distro_finder.DistroFinder
+    site_template = "site_distro_finder.json"
+
+    def test_content(self, distro_finder_info):
+        """Content always returns the parent of the provided path currently."""
+        finder = self.distro_finder_cls(distro_finder_info.root)
+        # We may want to improve this later, but it works for now
+        path = distro_finder_info.root / ".hab.json"
+        result = finder.content(path)
+        assert result == distro_finder_info.root
+
+    def test_installed(self, distro_finder_info, helpers, tmp_path):
+        self.check_installed(distro_finder_info, helpers, tmp_path)
+
+    def test_install(self, distro_finder_info, helpers, tmp_path):
+        self.check_install(distro_finder_info, helpers, tmp_path)
+
+
+class TestZipSidecar(CheckDistroFinder):
+    """Tests specific to `DistroFinderZip`."""
+
+    distro_finder_cls = zip_sidecar.DistroFinderZipSidecar
+    site_template = "site_distro_zip_sidecar.json"
+
+    def test_installed(self, zip_distro_sidecar, helpers, tmp_path):
+        self.check_installed(zip_distro_sidecar, helpers, tmp_path)
+
+    def test_install(self, zip_distro_sidecar, helpers, tmp_path):
+        self.check_install(zip_distro_sidecar, helpers, tmp_path)
+
+
+class TestZip(CheckDistroFinder):
+    """Tests specific to `DistroFinderZip`."""
+
+    distro_finder_cls = df_zip.DistroFinderZip
+    site_template = "site_distro_zip.json"
+
+    def test_content(self, zip_distro):
+        finder = df_zip.DistroFinderZip(zip_distro.root)
+        # If path is already a .zip file, it is just returned
+        path = zip_distro.root / "already_zip.zip"
+        result = finder.content(path)
+        assert result == path
+
+        # The right most .zip file is returned if path has multiple .zip suffixes.
+        path = zip_distro.root / "a.zip" / "b.zip"
+        result = finder.content(path)
+        assert result == path
+
+        # If a member path is passed, return the right most .zip suffix.
+        member_path = path / ".hab.json"
+        result = finder.content(member_path)
+        assert result == path
+
+        # member paths with nested return the right most .zip suffix.
+        member_path = path / "folder" / "sub-folder" / "file.json"
+        result = finder.content(member_path)
+        assert result == path
+
+        # If no .zip suffix is passed, the original path is returned.
+        path = zip_distro.root / "not_an_archive.txt"
+        result = finder.content(path)
+        assert result == path
+
+    def test_load_path(self, zip_distro):
+        """The Zip finder reads a .json file from inside the zip distro file.
+
+        Ensure it's able to read data from the .json file.
+        """
+        finder = df_zip.DistroFinderZip(zip_distro.root)
+
+        # This distro hard codes the version inside the .json file
+        data = finder.load_path(zip_distro.root / "dist_a_v0.1.zip")
+        assert data["name"] == "dist_a"
+        assert "distros" not in data
+        assert data["version"] == "0.1"
+
+        # Test a different distro that doesn't hard code the version
+        data = finder.load_path(zip_distro.root / "dist_b_v0.5.zip")
+        assert data["name"] == "dist_b"
+        assert "distros" not in data
+        assert data["version"] == "0.5"
+
+        # This distro includes required distros
+        data = finder.load_path(zip_distro.root / "dist_a_v0.2.zip")
+        assert data["name"] == "dist_a"
+        assert data["distros"] == ["dist_b"]
+        assert data["version"] == "0.2"
+
+    def test_zip_get_file_data(self, zip_distro, caplog):
+        """Test edge cases for `DistroFinderZip.get_file_data`."""
+        finder = df_zip.DistroFinderZip(zip_distro.root)
+        assert finder._cache == {}
+
+        # This file doesn't have a .hab.json file inside it
+        path = zip_distro.root / "not_valid_v0.1.zip"
+        data = finder.get_file_data(path)
+        assert data is None
+        assert [path / ".hab.json"] == list(finder._cache.keys())
+        finder.clear_cache()
+
+        # Check what happens if a member path isn't provided(Just the .zip file path)
+        path = zip_distro.root / "dist_a_v0.1.zip"
+        member_path = path / ".hab.json"
+        caplog.clear()
+        with caplog.at_level(logging.DEBUG, logger="hab.distro_finders.df_zip"):
+            data = finder.get_file_data(path)
+        check = [f'Implicitly added member ".hab.json" to path "{member_path}".']
+        assert check == [rec.message for rec in caplog.records]
+        # The raw data text was read and returned
+        assert data == b'{\n    "name": "dist_a",\n    "version": "0.1"\n}'
+        assert member_path in finder._cache
+
+        # Test that the cache is returned if populated
+        data = "Data already in the cache"
+        finder._cache[Path(member_path)] = data
+        assert finder.get_file_data(member_path) is data
+
+    def test_installed(self, zip_distro, helpers, tmp_path):
+        self.check_installed(zip_distro, helpers, tmp_path)
+
+    def test_install(self, zip_distro, helpers, tmp_path):
+        self.check_install(zip_distro, helpers, tmp_path)
+
+
+# TODO: Break this into separate smaller tests of components for each class not this
+@pytest.mark.parametrize(
+    "distro_info",
+    (
+        # "distro_finder_info",
+        "zip_distro",
+        "zip_distro_sidecar",
+    ),
+)
+def dtest_zip(request, distro_info, helpers, tmp_path):
+    # Convert the distro_info parameter to testing values.
+    df_cls = df_zip.DistroFinderZip
+    hab_json = ".hab.json"
+    implements_cache = True
+    parent_type = True
+    site_filename = "site_distro_zip.json"
+    if distro_info == "zip_distro_sidecar":
+        df_cls = zip_sidecar.DistroFinderZipSidecar
+        hab_json = "{name}_v{ver}.hab.json"
+        implements_cache = False
+        parent_type = "sidecar"
+        site_filename = "site_distro_zip_sidecar.json"
+    elif distro_info == "distro_finder_info":
+        df_cls = distro_finder.DistroFinder
+        implements_cache = False
+        parent_type = "directory"
+        site_filename = "site_distro_finder.json"
+    distro_info = request.getfixturevalue(distro_info)
+
+    site_file = tmp_path / "site.json"
+    helpers.render_template(
+        site_filename, site_file, zip_root=distro_info.root.as_posix()
+    )
+    site_distros = tmp_path / "distros"
+
+    check = set([v[:2] for v in distro_info.versions])
+
+    site = Site([site_file])
+    resolver = Resolver(site)
+    results = set()
+    # The correct class was resolved
+    df = resolver.distro_paths[0]
+    assert type(df) == df_cls
+
+    if implements_cache:
+        assert df._cache == {}
+
+    for node in resolver.dump_forest(resolver.distros, attr=None):
+        distro = node.node
+        if not isinstance(distro, DistroVersion):
+            continue
+
+        # Ensure the finder used to create this distro is set
+        assert distro.finder == df
+
+        assert distro.filename.name == hab_json.format(
+            name=distro.distro_name, ver=distro.version
+        )
+        if parent_type == "zip":
+            # If the parent is a zip, then the parent is a zip file
+            assert distro.filename.parent.suffix == ".zip"
+            assert distro.filename.parent.is_file()
+        elif parent_type == "sidecar":
+            # There is a sidecar zip file next to the *.hab.json file
+            zip_filename = distro.filename.name.replace(".hab.json", ".zip")
+            assert (distro.filename.parent / zip_filename).is_file()
+        elif parent_type == "directory":
+            assert distro.filename.is_file()
+            assert distro.filename.name == ".hab.json"
+
+        if implements_cache:
+            assert distro.filename in df._cache
+
+        results.add((distro.distro_name, str(distro.version)))
+
+        # Test the install process extracts all of the files from the zip
+        dest = site_distros / distro.distro_name / str(distro.version)
+        assert not dest.exists()
+        df.install(distro.filename, dest)
+        assert dest.is_dir()
+        assert (dest / ".hab.json").exists()
+        assert (dest / "file_a.txt").exists()
+        assert (dest / "folder/file_b.txt").exists()
+
+        # Test that if you try to install an already existing distro
+        # an exception is raised
+        with pytest.raises(
+            InstallDestinationExistsError, match="The destination already exists:"
+        ) as excinfo:
+            df.install(distro.filename, dest)
+        assert excinfo.value.filename == dest
+
+        # Test the installed function
+        # Returns True if passed a distro version folder containing a .hab.json
+        assert df.installed(dest)
+        # It returns False if the .hab.json file doesn't exist
+        assert not df.installed(site_distros)
+
+    if implements_cache:
+        df.clear_cache()
+        assert df._cache == {}
+
+    assert results == check
diff --git a/tests/test_parsing.py b/tests/test_parsing.py
index 4d81378..0be9f17 100644
--- a/tests/test_parsing.py
+++ b/tests/test_parsing.py
@@ -1,7 +1,9 @@
 import copy
 import json
+import pickle
 import re
 import sys
+from datetime import date, datetime
 from pathlib import Path
 
 import anytree
@@ -9,7 +11,8 @@
 import setuptools_scm
 from packaging.version import Version
 
-from hab import NotSet, utils
+from hab import NotSet, Resolver, utils
+from hab.distro_finders.distro_finder import DistroFinder
 from hab.errors import (
     DuplicateJsonError,
     HabError,
@@ -17,10 +20,10 @@
     ReservedVariableNameError,
     _IgnoredVersionError,
 )
-from hab.parsers import Config, DistroVersion, FlatConfig
+from hab.parsers import Config, DistroVersion, FlatConfig, HabBase
 
 
-class TestLoadJsonFile:
+class TestLoadJson:
     """Tests various conditions when using `hab.utils.load_json_file` to ensure
     expected output.
     """
@@ -34,16 +37,28 @@ def test_missing(self, tmpdir):
             utils.load_json_file(path)
         assert Path(excinfo.value.filename) == path
 
+    @classmethod
+    def check_exception(cls, excinfo, native_json, path):
+        if native_json:
+            # If built-in json was used, check that filename was appended to the message
+            assert f'Source("{path}")' in str(excinfo.value)
+        else:
+            # If pyjson5 was used, check that the filename was added to str
+            assert f"'source': {str(path)!r}" in str(excinfo.value)
+            # Check that the filename was added to the result dict
+            assert excinfo.value.result["source"] == str(path)
+
     def test_binary(self, tmpdir):
         """If attempting to read a binary file, filename is included in exception.
 
         This is a problem we run into rarely where a text file gets
         replaced/generated with a binary file containing noting but a lot of null bytes.
         """
+        bin_data = b"\x00" * 32
         path = Path(tmpdir) / "binary.json"
         # Create a binary test file containing multiple binary null values.
         with path.open("wb") as fle:
-            fle.write(b"\x00" * 32)
+            fle.write(bin_data)
 
         # Detect if using pyjson5 or not
         native_json = False
@@ -56,15 +71,15 @@ def test_binary(self, tmpdir):
         else:
             exc_type = pyjson5.pyjson5.Json5IllegalCharacter
 
+        # Test load_json_file
         with pytest.raises(exc_type) as excinfo:
             utils.load_json_file(path)
+        self.check_exception(excinfo, native_json, path)
 
-        if native_json:
-            # If built-in json was used, check that filename was appended to the message
-            assert f'Filename("{path}")' in str(excinfo.value)
-        else:
-            # If pyjson5 was used, check that the filename was added to the result dict
-            assert f"{{'filename': {str(path)!r}}}" in str(excinfo.value)
+        # Test loads_json
+        with pytest.raises(exc_type) as excinfo:
+            utils.loads_json(bin_data.decode(), path)
+        self.check_exception(excinfo, native_json, path)
 
     def test_config_load(self, uncached_resolver):
         cfg = Config({}, uncached_resolver)
@@ -77,6 +92,18 @@ def test_config_load(self, uncached_resolver):
         with pytest.raises(FileNotFoundError):
             cfg.load("invalid_path.json")
 
+    def test_loads_json(self, config_root):
+        """Test that `loads_json` is able to parse a valid json string."""
+        filename = config_root / "site_main.json"
+        with filename.open() as fle:
+            text = fle.read()
+        # Test an existing file is able to be parsed successfully.
+        data = utils.loads_json(text, filename)
+        # Spot check that we were able to parse data from the file.
+        assert isinstance(data, dict)
+        assert "append" in data
+        assert "set" in data
+
 
 def test_distro_parse(config_root, resolver):
     """Check that a distro json can be parsed correctly"""
@@ -182,13 +209,18 @@ def get_ver(*args, **kwargs):
             app.load(path)
 
 
-def test_distro_version(resolver):
+def test_distro_version(resolver, zip_distro_sidecar):
     """Verify that we find the expected version for a given requirement."""
     maya = resolver.distros["maya2020"]
 
     assert maya.latest_version("maya2020").name == "maya2020==2020.1"
     assert maya.latest_version("maya2020<2020.1").name == "maya2020==2020.0"
 
+    forest = {}
+    resolver = Resolver()
+    parsed = HabBase(forest, resolver, zip_distro_sidecar.root / "dist_a_v0.1.hab.json")
+    assert parsed.version == Version("0.1")
+
 
 def test_config_parse(config_root, resolver, helpers):
     """Check that a config json can be parsed correctly"""
@@ -291,6 +323,7 @@ def test_metaclass():
             "environment",
             "environment_config",
             "filename",
+            "finder",
             "min_verbosity",
             "name",
             "optional_distros",
@@ -686,19 +719,16 @@ def test_invalid_config(config_root, resolver):
 
     with pytest.raises(_JsonException) as excinfo:
         Config({}, resolver, filename=path)
-
-    if native_json:
-        # If built-in json was used, check that filename was appended to the message
-        assert f'Filename("{path}")' in str(excinfo.value)
-    else:
-        # If pyjson5 was used, check that the filename was added to the result dict
-        assert excinfo.value.result["filename"] == str(path)
+    TestLoadJson.check_exception(excinfo, native_json, path)
 
 
 def test_misc_coverage(resolver):
     """Test that cover misc lines not covered by the rest of the tests"""
     assert str(NotSet) == "NotSet"
     assert copy.copy(NotSet) is NotSet
+    # Check that NotSet can be pickled
+    payload = pickle.dumps(NotSet)
+    assert pickle.loads(payload) is NotSet
 
     # Check that dirname is modified when setting a blank filename
     cfg = Config({}, resolver)
@@ -761,10 +791,13 @@ def test_duplicated_distros(config_root, resolver):
     definitions are in the same config_path so a DuplicateJsonError is raised.
     """
     original = resolver.distro_paths
+    site = resolver.site
 
     # Check that the first config in distro_paths was used
     distro_paths = list(original)
-    distro_paths.insert(0, config_root / "duplicates" / "distros_1" / "*")
+    distro_paths.insert(
+        0, DistroFinder(config_root / "duplicates" / "distros_1" / "*", site=site)
+    )
     resolver.distro_paths = distro_paths
 
     dcc = resolver.find_distro("the_dcc==1.2")
@@ -774,7 +807,9 @@ def test_duplicated_distros(config_root, resolver):
     # Check that an exception is raised if there are duplicate definitions from
     # the same distro_paths directory.
     distro_paths = list(original)
-    distro_paths.insert(0, config_root / "duplicates" / "distros_2" / "*")
+    distro_paths.insert(
+        0, DistroFinder(config_root / "duplicates" / "distros_2" / "*", site=site)
+    )
     resolver.distro_paths = distro_paths
 
     with pytest.raises(DuplicateJsonError):
@@ -1172,7 +1207,7 @@ def test_reserved(self, uncached_resolver, variables, invalid, tmpdir):
 
         # Add the test distro to hab's distro search. We don't need to call
         # `clear_caches` because distros haven't been resolved yet.
-        uncached_resolver.distro_paths.append(Path(tmpdir))
+        uncached_resolver.distro_paths.append(DistroFinder(Path(tmpdir)))
 
         # When distros are resolved, an exception should be raised
         with pytest.raises(
@@ -1180,3 +1215,14 @@ def test_reserved(self, uncached_resolver, variables, invalid, tmpdir):
             match=rf"'{invalid}' are reserved variable name\(s\) for hab",
         ):
             uncached_resolver.distros
+
+
+def test_hab_json_encoder():
+    # These non-standard data types are supported by HabJsonEncoder
+    json.dumps(NotSet, indent=4, cls=utils.HabJsonEncoder)
+    json.dumps(datetime.now(), indent=4, cls=utils.HabJsonEncoder)
+    json.dumps(date.today(), indent=4, cls=utils.HabJsonEncoder)
+
+    # Errors are still raised if passing un-handled data types.
+    with pytest.raises(TypeError):
+        json.dumps(object, indent=4, cls=utils.HabJsonEncoder)
diff --git a/tests/test_resolver.py b/tests/test_resolver.py
index 4a073b4..18bd1b4 100644
--- a/tests/test_resolver.py
+++ b/tests/test_resolver.py
@@ -3,13 +3,16 @@
 import sys
 from collections import OrderedDict
 from pathlib import Path
+from zipfile import ZipFile
 
 import anytree
 import pytest
 from packaging.requirements import Requirement
 
-from hab import NotSet, Resolver, Site, utils
+from hab import DistroMode, NotSet, Resolver, Site, utils
+from hab.distro_finders.distro_finder import DistroFinder
 from hab.errors import InvalidRequirementError
+from hab.parsers import DistroVersion
 from hab.solvers import Solver
 
 
@@ -17,12 +20,14 @@ def test_environment_variables(config_root, helpers, monkeypatch):
     """Check that Resolver's init respects the environment variables it uses."""
     config_paths_env = utils.Platform.expand_paths(["a/config/path", "b/config/path"])
     distro_paths_env = utils.Platform.expand_paths(["a/distro/path", "b/distro/path"])
+    distro_paths_env = [DistroFinder(p) for p in distro_paths_env]
     config_paths_direct = utils.Platform.expand_paths(
         ["z/config/path", "zz/config/path"]
     )
     distro_paths_direct = utils.Platform.expand_paths(
         ["z/distro/path", "zz/distro/path"]
     )
+    distro_paths_direct = [DistroFinder(p) for p in distro_paths_direct]
 
     # Set the config environment variables
     monkeypatch.setenv(
@@ -116,6 +121,52 @@ def test_closest_config(resolver, path, result, reason):
     assert resolver.closest_config(path).fullpath == result, reason
 
 
+def test_distro_mode(zip_distro, helpers, tmp_path):
+    """Test `Resolver.distro_mode` is respected when calling `distros`.
+
+    Also test that the `distro_mode_override` with context updates and restores
+    the distro_mode.
+    """
+    site_file = tmp_path / "site.json"
+    helpers.render_template(
+        "site_download.json", site_file, zip_root=zip_distro.root.as_posix()
+    )
+    resolver = Resolver(Site([site_file]))
+
+    # Install some distros so the resolver can find them
+    for distro, version in (("dist_a", "0.1"), ("dist_b", "0.5")):
+        with ZipFile(zip_distro.root / f"{distro}_v{version}.zip") as zip_info:
+            zip_info.extractall(tmp_path / "distros" / distro / version)
+
+    def get_dist_names():
+        return [
+            row.node.name
+            for row in resolver.dump_forest(resolver.distros, attr=None)
+            if isinstance(row.node, DistroVersion)
+        ]
+
+    # Get the installed distros and check that `.distros` is the correct return
+    installed = get_dist_names()
+    assert resolver.distros is resolver._installed_distros
+    # Get the download distros and check that `.distros` is the correct return
+    with resolver.distro_mode_override(DistroMode.Downloaded):
+        downloads = get_dist_names()
+        assert resolver.distros is resolver._downloadable_distros
+    # Check that the installed distros are accessible again
+    installed_after = get_dist_names()
+    assert resolver.distros is resolver._installed_distros
+
+    assert installed == ["dist_a==0.1", "dist_b==0.5"]
+    assert installed_after == installed
+    assert downloads == [
+        "dist_a==0.1",
+        "dist_a==0.2",
+        "dist_a==1.0",
+        "dist_b==0.5",
+        "dist_b==0.6",
+    ]
+
+
 class TestDumpForest:
     """Test the dump_forest method on resolver"""
 
@@ -958,19 +1009,19 @@ def test_clear_caches(resolver):
     """Test that Resolver.clear_cache works as expected."""
     # Resolver cache is empty
     assert resolver._configs is None
-    assert resolver._distros is None
+    assert resolver._installed_distros is None
 
     # Populate resolver cache data
     resolver.resolve("not_set")
     assert isinstance(resolver._configs, dict)
-    assert isinstance(resolver._distros, dict)
+    assert isinstance(resolver._installed_distros, dict)
     assert len(resolver._configs) > 1
-    assert len(resolver._distros) > 1
+    assert len(resolver._installed_distros) > 1
 
     # Calling clear_caches resets the resolver cache
     resolver.clear_caches()
     assert resolver._configs is None
-    assert resolver._distros is None
+    assert resolver._installed_distros is None
 
 
 def test_clear_caches_cached(habcached_resolver):
diff --git a/tests/test_site.py b/tests/test_site.py
index b60d3b8..ee2a83f 100644
--- a/tests/test_site.py
+++ b/tests/test_site.py
@@ -1,4 +1,5 @@
 import sys
+import tempfile
 from pathlib import Path, PurePosixPath, PureWindowsPath
 
 import pytest
@@ -6,6 +7,7 @@
 
 from hab import Resolver, Site, utils
 from hab.cache import Cache
+from hab.distro_finders.distro_finder import DistroFinder
 
 
 def test_environment_variables(config_root, monkeypatch):
@@ -206,7 +208,7 @@ def test_paths(self, config_root, helpers):
         )
         assert len(site.get("distro_paths")) == 2
         helpers.check_path_list(
-            site.get("distro_paths"),
+            [p.root for p in site.get("distro_paths")],
             (
                 config_root / "distros" / "*",
                 config_root / "duplicates" / "distros_1" / "*",
@@ -228,7 +230,7 @@ def test_paths_reversed(self, config_root, helpers):
         )
         assert len(site.get("distro_paths")) == 2
         helpers.check_path_list(
-            site.get("distro_paths"),
+            [p.root for p in site.get("distro_paths")],
             (
                 config_root / "duplicates" / "distros_1" / "*",
                 config_root / "distros" / "*",
@@ -289,8 +291,8 @@ def test_dump_cached(config_root, habcached_site_file):
         f"            {other_site}",
         f"{{green}}config_paths:  {{reset}}config\\path\\{platform}",
         f"               {config_root}\\configs\\*{{cached}}",
-        f"{{green}}distro_paths:  {{reset}}distro\\path\\{platform}",
-        f"               {config_root}\\distros\\*{{cached}}",
+        f"{{green}}distro_paths:  {{reset}}distro\\path\\{platform}{{cls_name}}",
+        f"               {config_root}\\distros\\*{{cls_name}}{{cached}}",
     )
     check_template = "\n".join(check_template)
     colors = {
@@ -304,13 +306,22 @@ def test_dump_cached(config_root, habcached_site_file):
     # No verbosity, should not show cached status
     assert site.get("colorize") is None
     result = site.dump(width=60)
-    check = check_template.format(cached="", **colors)
+    check = check_template.format(cached="", cls_name="", **colors)
     assert check in result
 
     # verbosity enabled, should show cached status
     result = site.dump(verbosity=1, width=60)
     check = check_template.format(
-        cached=f" {Fore.YELLOW}(cached){Style.RESET_ALL}", **colors
+        cached=f" {Fore.YELLOW}(cached){Style.RESET_ALL}", cls_name="", **colors
+    )
+    assert check in result
+
+    # verbosity level 2, should also show DistroFinder classes
+    result = site.dump(verbosity=2, width=60)
+    check = check_template.format(
+        cached=f" {Fore.YELLOW}(cached){Style.RESET_ALL}",
+        cls_name=f" {Fore.CYAN}[DistroFinder]{Style.RESET_ALL}",
+        **colors,
     )
     assert check in result
 
@@ -320,12 +331,19 @@ def test_dump_cached(config_root, habcached_site_file):
 
     # No verbosity, should not show cached status
     result = site.dump(width=60)
-    check = check_template.format(cached="", green="", reset="")
+    check = check_template.format(cached="", green="", reset="", cls_name="")
     assert check in result
 
     # verbosity enabled, should show cached status
     result = site.dump(verbosity=1, width=60)
-    check = check_template.format(cached=" (cached)", green="", reset="")
+    check = check_template.format(cached=" (cached)", green="", reset="", cls_name="")
+    assert check in result
+
+    # verbosity level 2, should also show DistroFinder classes
+    result = site.dump(verbosity=2, width=60)
+    check = check_template.format(
+        cached=" (cached)", green="", reset="", cls_name=" [DistroFinder]"
+    )
     assert check in result
 
 
@@ -340,7 +358,7 @@ def test_linux(self, monkeypatch, config_root):
         site = Site(paths)
 
         assert site.get("config_paths") == [Path("config/path/linux")]
-        assert site.get("distro_paths") == [Path("distro/path/linux")]
+        assert site.get("distro_paths") == [DistroFinder(Path("distro/path/linux"))]
         assert site.get("platforms") == ["windows", "linux"]
 
     def test_osx(self, monkeypatch, config_root):
@@ -353,7 +371,7 @@ def test_osx(self, monkeypatch, config_root):
         site = Site(paths)
 
         assert site.get("config_paths") == [Path("config/path/osx")]
-        assert site.get("distro_paths") == [Path("distro/path/osx")]
+        assert site.get("distro_paths") == [DistroFinder(Path("distro/path/osx"))]
         assert site.get("platforms") == ["osx", "linux"]
 
     def test_win(self, monkeypatch, config_root):
@@ -366,7 +384,7 @@ def test_win(self, monkeypatch, config_root):
         site = Site(paths)
 
         assert site.get("config_paths") == [Path("config\\path\\windows")]
-        assert site.get("distro_paths") == [Path("distro\\path\\windows")]
+        assert site.get("distro_paths") == [DistroFinder(Path("distro\\path\\windows"))]
         assert site.get("platforms") == ["windows", "osx"]
 
 
@@ -756,3 +774,93 @@ def test_habcache_cls(self, config_root, uncached_resolver):
             match="hab_test_entry_points.CacheVX class was used",
         ):
             Site([config_root / "site" / "eps" / "site_habcache_cls.json"])
+
+    def test_entry_point_init(self, config_root):
+        site = Site([config_root / "site_main.json"])
+        instance = site.entry_point_init(
+            "group.name",
+            "hab.distro_finders.distro_finder:DistroFinder",
+            ["a/root/path", {"site": "a Site Instance"}],
+        )
+        # The entry_point class was imported and initialized
+        assert isinstance(instance, DistroFinder)
+        # The instance had the requested arguments passed to it
+        assert instance.root == Path("a/root/path")
+        # The last item was a dictionary, that was removed from args and passed
+        # as kwargs.
+        # NOTE: you should not pass site using this method. It's being used here
+        # to test the kwargs feature and ensure the default site setting doesn't
+        # overwrite site if it was passed as a kwarg.
+        assert instance.site == "a Site Instance"
+
+        # Don't pass a kwargs dict, it should get site from itself.
+        instance = site.entry_point_init(
+            "group.name",
+            "hab.distro_finders.distro_finder:DistroFinder",
+            ["b/root/path"],
+        )
+        assert instance.root == Path("b/root/path")
+        assert instance.site is site
+
+
+class TestDownloads:
+    # Defaults to `$TEMP/hab_downloads` if not specified
+    default_cache_root = Path(tempfile.gettempdir()) / "hab_downloads"
+
+    def test_download_cache(self, config_root, uncached_resolver):
+        """Test how `site.downloads["cache_root"]` is processed."""
+        site = uncached_resolver.site
+        assert site.downloads["cache_root"] == self.default_cache_root
+        # `Platform.default_download_cache()` returns the expected default value
+        assert utils.Platform.default_download_cache() == self.default_cache_root
+
+        # If specified, only the first path is used. This is using a non-valid
+        # relative path for testing, in practice this should be a absolute path.
+        paths = [config_root / "site" / "site_distro_finder.json"]
+        site = Site(paths)
+        assert (
+            site.downloads["cache_root"] == Path("hab testable") / "download" / "path"
+        )
+
+        # Use the default if site specifies cache_root but its an empty string.
+        paths = [config_root / "site" / "site_distro_finder_empty.json"]
+        site = Site(paths)
+        assert site.downloads["cache_root"] == self.default_cache_root
+
+    def test_lazy(self, config_root):
+        site = Site([config_root / "site" / "site_distro_finder.json"])
+        # Check that downloads is not parsed before the downloads property
+        # is first called.
+        assert site._downloads_parsed is False
+        downloads = site.downloads
+        assert site._downloads_parsed is True
+        assert site.downloads is downloads
+
+    def test_default_settings(self, config_root):
+        """Test the default downloads values if not defined by site files."""
+        site = Site([config_root / "site_main.json"])
+        downloads = site.downloads
+        assert len(downloads["distros"]) == 0
+
+        # cache_root is always defined
+        assert downloads["cache_root"] == self.default_cache_root
+        # These are only defined if the json file defines them.
+        assert "install_root" not in downloads
+        assert "relative_path" not in downloads
+
+    def test_all_settings_defined(self, config_root):
+        """Test the resolved downloads values defined by a site file."""
+        from hab.distro_finders.df_zip import DistroFinderZip
+
+        site = Site([config_root / "site" / "site_distro_finder.json"])
+        downloads = site.downloads
+
+        # Check that each part of downloads was processed correctly
+        assert len(downloads["distros"]) == 1
+        finder = downloads["distros"][0]
+        assert isinstance(finder, DistroFinderZip)
+        assert finder.root == Path("network_server/distro/source")
+
+        assert downloads["cache_root"] == Path("hab testable/download/path")
+        assert downloads["install_root"] == config_root / "site" / "distros"
+        assert downloads["relative_path"] == "{distro_name}_v{version}"