4/4: Add testing for the new hab install and DistroFinder features

hab/utils.py

@@ -6,6 +6,7 @@
 import os
 import re
 import sys
+import tempfile
 import textwrap
 import zlib
 from abc import ABC, abstractmethod
@@ -542,6 +543,14 @@ def default_ext(cls):
         """Returns the default file extension used on this platform."""
         return cls._default_ext
 
+    @classmethod
+    def default_download_cache(cls):
+        """Path where download files are cached.
+
+        This is used as the default location for `Site.downloads["cache_root"]`.
+        """
+        return Path(tempfile.gettempdir()) / "hab_downloads"
+
     @classmethod
     def expand_paths(cls, paths):
         """Converts path strings separated by ``cls.pathsep()`` and lists into

tests/conftest.py

@@ -1,9 +1,13 @@
 import json
 import os
+import shutil
+from collections import namedtuple
 from contextlib import contextmanager
 from pathlib import Path, PurePath
+from zipfile import ZipFile
 
 import pytest
+from jinja2 import Environment, FileSystemLoader
 from packaging.requirements import Requirement
 
 from hab import Resolver, Site
@@ -111,6 +115,93 @@ def resolver(request):
     return request.getfixturevalue(test_map[request.param])
 
 
+Distro = namedtuple("Distro", ["name", "version", "inc_version", "distros"])
+
+
+class DistroInfo(namedtuple("DistroInfo", ["root", "versions"])):
+    default_versions = (
+        ("dist_a", "0.1", True, None),
+        ("dist_a", "0.2", False, ["dist_b"]),
+        ("dist_a", "1.0", False, None),
+        ("dist_b", "0.5", False, None),
+        ("dist_b", "0.6", False, None),
+    )
+
+    @classmethod
+    def dist_version(cls, distro, version):
+        return f"{distro}_v{version}"
+
+    @classmethod
+    def hab_json(cls, distro, version=None, distros=None):
+        data = {"name": distro}
+        if version:
+            data["version"] = version
+        if distros:
+            data["distros"] = distros
+        return json.dumps(data, indent=4)
+
+    @classmethod
+    def generate(cls, root, versions=None, zip_created=None):
+        if versions is None:
+            versions = cls.default_versions
+
+        versions = {(x[0], x[1]): Distro(*x) for x in versions}
+
+        for version in versions.values():
+            name = cls.dist_version(version.name, version.version)
+            filename = root / f"{name}.zip"
+            ver = version.version if version.inc_version else None
+            with ZipFile(filename, "w") as zf:
+                zf.writestr(
+                    ".hab.json",
+                    cls.hab_json(version.name, version=ver, distros=version.distros),
+                )
+                zf.writestr("file_a.txt", "File A inside the distro.")
+                zf.writestr("folder/file_b.txt", "File B inside the distro.")
+                if zip_created:
+                    zip_created(zf)
+
+        # Create a correctly named .zip file that doesn't have a .hab.json file
+        # to test for .zip files that are not distros.
+        with ZipFile(root / "not_valid_v0.1.zip", "w") as zf:
+            zf.writestr("README.txt", "This file is not a hab distro zip.")
+
+        return cls(root, versions)
+
+
+@pytest.fixture(scope="session")
+def zip_distro(tmp_path_factory):
+    """Returns a DistroInfo instance for a zip folder structure.
+
+    This is useful if the zip files are locally accessible or if your hab download
+    server supports `HTTP range requests`_. For example if you are using Amazon S3.
+
+    .. _HTTP range requests:
+       https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests
+    """
+    root = tmp_path_factory.mktemp("_zip_distro")
+    return DistroInfo.generate(root)
+
+
+@pytest.fixture(scope="session")
+def zip_distro_sidecar(tmp_path_factory):
+    """Returns a DistroInfo instance for a zip folder structure with sidecar
+    `.hab.json` files.
+
+    This is useful when your hab download server does not support HTTP range requests.
+    """
+    root = tmp_path_factory.mktemp("_zip_distro_sidecar")
+
+    def zip_created(zf):
+        """Extract the .hab.json from the zip to a sidecar file."""
+        filename = Path(zf.filename).stem
+        sidecar = root / f"{filename}.hab.json"
+        path = zf.extract(".hab.json", root)
+        shutil.move(path, sidecar)
+
+    return DistroInfo.generate(root, zip_created=zip_created)
+
+
 class Helpers(object):
     """A collection of reusable functions that tests can use."""
 
@@ -204,6 +295,19 @@ def compare_files(generated, check):
                 cache[i] == check[i]
             ), f"Difference on line: {i} between the generated cache and {generated}."
 
+    @staticmethod
+    def render_template(template, dest, **kwargs):
+        environment = Environment(
+            loader=FileSystemLoader(str(Path(__file__).parent / "templates")),
+            trim_blocks=True,
+            lstrip_blocks=True,
+        )
+        template = environment.get_template(template)
+
+        text = template.render(**kwargs).rstrip() + "\n"
+        with dest.open("w") as fle:
+            fle.write(text)
+
 
 @pytest.fixture
 def helpers():

tests/site/site_distro_finder.json

@@ -0,0 +1,32 @@
+{
+    "set":
+    {
+        "distro_paths":
+        [
+            [
+                "hab.distro_finders.distro_finder:DistroFinder",
+                "hab testable/download/path"
+            ],
+            [
+                "hab.distro_finders.distro_finder:DistroFinder",
+                "hab testing/downloads",
+                {
+                    "site": "for testing only, do not specify site"
+                }
+            ]
+        ],
+        "downloads":
+        {
+            "cache_root": "hab testable/download/path",
+            "distros":
+            [
+                [
+                    "hab.distro_finders.df_zip:DistroFinderZip",
+                    "network_server/distro/source"
+                ]
+            ],
+            "install_root": "{relative_root}/distros",
+            "relative_path": "{{distro_name}}_v{{version}}"
+        }
+    }
+}

tests/site/site_distro_finder_empty.json

@@ -0,0 +1,7 @@
+{
+    "set": {
+        "downloads": {
+            "cache_root": ""
+        }
+    }
+}

tests/templates/site_distro_zip.json

@@ -0,0 +1,12 @@
+{
+    "set":
+    {
+        "distro_paths":
+        [
+            [
+                "hab.distro_finders.df_zip:DistroFinderZip",
+                "{{ zip_root }}"
+            ]
+        ]
+    }
+}

tests/templates/site_distro_zip_sidecar.json

@@ -0,0 +1,12 @@
+{
+    "set":
+    {
+        "distro_paths":
+        [
+            [
+                "hab.distro_finders.zip_sidecar:DistroFinderZipSidecar",
+                "{{ zip_root }}"
+            ]
+        ]
+    }
+}

tests/templates/site_download.json

@@ -0,0 +1,20 @@
+{
+    "set": {
+        "config_paths": [
+            "{relative_root}/configs"
+        ],
+        "distro_paths": [
+            "{relative_root}/distros/*"
+        ],
+        "downloads": {
+            "cache_root": "{relative_root}/downloads",
+            "distros": [
+                [
+                    "hab.distro_finders.df_zip:DistroFinderZip",
+                    "{{ zip_root }}"
+                ]
+            ],
+            "install_root": "{relative_root}/distros"
+        }
+    }
+}

tests/test_distro_finder.py

@@ -0,0 +1,144 @@
+import glob
+from pathlib import Path
+
+import pytest
+
+from hab import Resolver, Site, utils
+from hab.distro_finders import df_zip, distro_finder, zip_sidecar
+from hab.parsers import DistroVersion
+
+
+def test_distro_finder_entry_point(config_root):
+    """Test edge cases for DistroFinder entry_point processing."""
+    paths = [config_root / "site" / "site_distro_finder.json"]
+    site = Site(paths)
+    distro_paths = site["distro_paths"]
+    # Ensure the DistroFinder paths are set correctly when set as EntryPoint
+    assert distro_paths[0].root == Path("hab testable") / "download" / "path"
+    assert distro_paths[1].root == Path("hab testing") / "downloads"
+    # The second path passes the kwargs dict with `site`. This triggers testing
+    # when a dict is passed to the entry_point. However site is always set to
+    # the current site after a DistroFinder is initialized.
+    assert distro_paths[1].site == site
+
+
+def test_eq():
+    a = distro_finder.DistroFinder("path/a")
+
+    assert a == distro_finder.DistroFinder("path/a")
+    assert a != distro_finder.DistroFinder("path/b")
+
+    # Test that if the glob_str is different it will not compare equal
+    b = distro_finder.DistroFinder("path/a")
+    b.glob_str = "*/test.json"
+    assert a != b
+    # Test that if glob_str attr is missing it will not compare equal
+    del b.glob_str
+    assert a != b
+    # Restore glob_str and the objects will compare equal again
+    b.glob_str = "*/.hab.json"
+    assert a == b
+
+    # Test that if the root is different it will not compare equal
+    b.root = Path(".")
+    assert a != b
+    # Test that if root attr is missing it will not compare equal
+    del b.root
+    assert a != b
+    # Restore root and the objects will compare equal again
+    b.root = Path("path/a")
+    assert a == b
+
+
+@pytest.mark.parametrize(
+    "glob_str,count",
+    (
+        ("{root}/reference*/sh_*", 12),
+        ("{root}/reference/*", 0),
+        ("{root}/reference_scripts/*/*.sh", 20),
+    ),
+)
+def test_glob_path(config_root, glob_str, count):
+    """Ensure `hab.utils.glob_path` returns the expected results."""
+    glob_str = glob_str.format(root=config_root)
+    # Check against the `glob.glob` result.
+    check = sorted([Path(p) for p in glob.glob(glob_str)])
+
+    path_with_glob = Path(glob_str)
+    result = sorted(utils.glob_path(path_with_glob))
+
+    assert result == check
+    # Sanity check to ensure that the expected results were found by `glob.glob`
+    assert len(result) == count
+
+
+@pytest.mark.parametrize("distro_info", ("zip_distro", "zip_distro_sidecar"))
+def test_zip(request, distro_info, helpers, tmp_path):
+    # Convert the distro_info parameter to testing values.
+    df_cls = df_zip.DistroFinderZip
+    hab_json = ".hab.json"
+    implements_cache = True
+    parent_is_zip = True
+    site_filename = "site_distro_zip.json"
+    if distro_info == "zip_distro_sidecar":
+        df_cls = zip_sidecar.DistroFinderZipSidecar
+        hab_json = "{name}_v{ver}.hab.json"
+        implements_cache = False
+        parent_is_zip = False
+        site_filename = "site_distro_zip_sidecar.json"
+    distro_info = request.getfixturevalue(distro_info)
+
+    site_file = tmp_path / "site.json"
+    helpers.render_template(
+        site_filename, site_file, zip_root=distro_info.root.as_posix()
+    )
+    site_distros = tmp_path / "distros"
+
+    check = set([v[:2] for v in distro_info.versions])
+
+    site = Site([site_file])
+    resolver = Resolver(site)
+    results = set()
+    # The correct class was resolved
+    distro_finder = resolver.distro_paths[0]
+    assert type(distro_finder) == df_cls
+
+    if implements_cache:
+        assert distro_finder._cache == {}
+
+    for node in resolver.dump_forest(resolver.distros, attr=None):
+        distro = node.node
+        if not isinstance(distro, DistroVersion):
+            continue
+
+        assert distro.filename.name == hab_json.format(
+            name=distro.distro_name, ver=distro.version
+        )
+        if parent_is_zip:
+            # If the parent is a zip, then the parent is a zip file
+            assert distro.filename.parent.suffix == ".zip"
+            assert distro.filename.parent.is_file()
+        else:
+            # Otherwise there is a sidecar zip file next to the *.hab.json file
+            zip_filename = distro.filename.name.replace(".hab.json", ".zip")
+            assert (distro.filename.parent / zip_filename).is_file()
+
+        if implements_cache:
+            assert distro.filename in distro_finder._cache
+
+        results.add((distro.distro_name, str(distro.version)))
+
+        # Test the install process extracts all of the files from the zip
+        dest = site_distros / distro.distro_name / str(distro.version)
+        assert not dest.exists()
+        distro_finder.install(distro.filename, dest)
+        assert dest.is_dir()
+        assert (dest / ".hab.json").exists()
+        assert (dest / "file_a.txt").exists()
+        assert (dest / "folder/file_b.txt").exists()
+
+    if implements_cache:
+        distro_finder.clear_cache()
+        assert distro_finder._cache == {}
+
+    assert results == check