Add entry_points for customizing site configuration

- `hab.site.add_paths` Is run after `HAB_PATHS` is processed adding
additional paths to the site configuration.
- `hab.site.finalize` is run after site if fully resolved.

This allows for loading site config files dynamically. For example from
inside of a pip package that may not be stored in a standard location
but can be imported by python.

README.md

@@ -357,6 +357,15 @@ rules to keep in mind.
 on the outside of the the right site file's paths.
 3. For `platform_path_maps`, only the first key is kept and any duplicates
 are discarded.
+4. The entry_point `hab.site.add_paths` is processed separately after `HAB_PATH`
+or `--site` paths are processed, so:
+   * Each path added is treated as left most when merging into the final configuration.
+   * The entry_point `hab.site.add_paths` will be ignored for dynamically added paths.
+   * This can be used to include site files from inside of pip packages. For
+   example a host installed pip package may be installed in the system python,
+   or as a pip editable installation.
+   * Duplicate paths added dynamically are discarded keeping the first
+   encountered(right most).
 
 See [Defining Environments](#defining-environments) for how to structure the json
 to prepend, append, set, unset values.
@@ -441,6 +450,8 @@ for details on each item.
 | hab.cfg.reduce.env | Used to make any modifications to a config after the global env is resolved but before aliases are resolved. | `cfg` |  | [All][tt-multi-all] |
 | hab.cfg.reduce.finalize | Used to make any modifications to a config after aliases are resolved and just before the the config finishes reducing. | `cfg` |  | [All][tt-multi-all] |
 | hab.launch_cls | Used as the default `cls` by `hab.parsers.Config.launch()` to launch aliases from inside of python. This should be a subclass of subprocess.Popen. A [complex alias](#complex-aliases) may override this per alias. Defaults to [`hab.launcher.Launcher`](hab/launcher.py). [Example](tests/site/site_entry_point_a.json) |  |  | [First][tt-multi-first] |
+| hab.site.add_paths | Dynamically prepends extra [site configuration files](#site) to the current configuration. This entry_point is ignored for any configs added using this entry_point. | `site` | A `list` of `pathlib.Path` for existing site .json files. | [All][tt-multi-all] |
+| hab.site.finalize | Used to modify site configuration files just before the site is fully initialized. | `site` |  | [All][tt-multi-all] |
 | hab.uri.validate | Used to validate and modify a URI. If the URI is invalid, this should raise an exception. If the URI should be modified, then return the modified URI as a string. | `resolver`, `uri` | Updated URI as string or None. | [All][tt-multi-all] |
 
 The name of each entry point is used to de-duplicate results from multiple site json files.

hab/site.py

@@ -127,12 +127,37 @@ def load(self):
         """Iterates over each file in self.path. Replacing the value of each key.
         The last file in the list will have its settings applied even if other
         files define them."""
+        # Process the main site files. These are the only ones that can add the
+        # `hab.site.add_paths` entry_points.
         for path in reversed(self.paths):
             self.load_file(path)
 
+        # Now that the main site files are handle `hab.site.add_paths` entry_points.
+        # This lets you add site json files where you can't hard code the path.
+        # For example if you want a site file included in a pip package installed
+        # on a host, the path would change depending on the python version being
+        # used and if using a editable pip install.
+        for ep in self.entry_points_for_group("hab.site.add_paths"):
+            logger.debug(f"Running hab.site.add_paths entry_point: {ep}")
+            func = ep.load()
+
+            # This entry_point needs to return a list of site file paths as
+            # `pathlib.Path` records.
+            paths = func(site=self)
+            for path in reversed(paths):
+                if path in self.paths:
+                    logger.debug(f"Path already added, skipping: {path}")
+                    continue
+                logger.debug(f"Path added by hab.site.add_paths: {path}")
+                self.paths.insert(0, path)
+                self.load_file(path)
+
         # Ensure any platform_path_maps are converted to pathlib objects.
         self.standardize_platform_path_maps()
 
+        # Entry_point to allow modification as a final step of loading site files
+        self.run_entry_points_for_group("hab.site.finalize", site=self)
+
     def load_file(self, filename):
         """Load an individual file path and merge its contents onto self.
 

tests/hab_test_entry_points.py

@@ -25,6 +25,31 @@ def cfg_reduce_finalize(cfg):
     )
 
 
+def site_add_paths(site):
+    """Add a couple of extra site paths to hab using `hab.site.add_paths` entry_point."""
+    from pathlib import Path
+
+    return [
+        Path(__file__).parent / "site" / "eps" / "site_add_paths_a.json",
+        Path(__file__).parent / "site" / "eps" / "site_add_paths_b.json",
+    ]
+
+
+def site_add_paths_a(site):
+    """Add a couple of extra site paths to hab using `hab.site.add_paths` entry_point."""
+    from pathlib import Path
+
+    return [
+        Path(__file__).parent / "site" / "eps" / "site_add_paths_c.json",
+    ]
+
+
+def site_finalize(site):
+    """Used to test that an entry point is called by raising an exception when
+    called. See `tests/site/eps/README.md` for details."""
+    raise NotImplementedError("hab_test_entry_points.site_finalize called successfully")
+
+
 def uri_validate_error(resolver, uri):
     """Used to test that an entry point is called by raising an exception when
     called. See `tests/site/eps/README.md` for details."""

tests/site/eps/site_add_paths.json

@@ -0,0 +1,12 @@
+{
+    "set": {
+        "test_data": "site_add_paths.json"
+    },
+    "append": {
+        "entry_points": {
+            "hab.site.add_paths": {
+                "main": "hab_test_entry_points:site_add_paths"
+            }
+        }
+    }
+}

tests/site/eps/site_add_paths_a.json

@@ -0,0 +1,13 @@
+{
+    "set": {
+        "test_data": "site_add_paths_a.json"
+    },
+    "append": {
+        "entry_points": {
+            "hab.site.add_paths": {
+                "main": "hab_test_entry_points:site_add_paths",
+                "a": "hab_test_entry_points:site_add_paths_a"
+            }
+        }
+    }
+}

tests/site/eps/site_add_paths_b.json

@@ -0,0 +1,5 @@
+{
+    "set": {
+        "test_data": "site_add_paths_b.json"
+    }
+}

tests/site/eps/site_add_paths_c.json

@@ -0,0 +1,5 @@
+{
+    "set": {
+        "test_data": "site_add_paths_c.json"
+    }
+}

tests/site/eps/site_finalize.json

@@ -0,0 +1,9 @@
+{
+    "append": {
+        "entry_points": {
+            "hab.site.finalize": {
+                "main": "hab_test_entry_points:site_finalize"
+            }
+        }
+    }
+}

tests/test_site.py

@@ -554,3 +554,65 @@ def test_called_by_resolve(self, config_root, site_file, except_match):
         # The module has now been imported and the correct function was loaded
         with pytest.raises(NotImplementedError, match=except_match):
             resolver.resolve("default")
+
+    def test_site_add_paths_non_recursive(self, config_root):
+        """Checks that the `hab.site.add_paths` entry_point is respected for
+        file paths passed to the paths argument of Site. Also test that the
+        entry_point is ignored when processing these dynamically added paths.
+        """
+        site = Site(
+            [
+                config_root / "site" / "eps" / "site_add_paths.json",
+            ]
+        )
+
+        # Check that static and dynamic paths were added in the correct order.
+        assert len(site.paths) == 3
+        assert site.paths[0].name == "site_add_paths_a.json"
+        assert site.paths[1].name == "site_add_paths_b.json"
+        assert site.paths[2].name == "site_add_paths.json"
+
+        # Check which "set" value was resolved by the end. To correctly process
+        # the list returned by the entry_points are processed in reverse order
+        assert site["test_data"] == ["site_add_paths_a.json"]
+
+    def test_site_add_paths_multiple(self, config_root):
+        """Checks that multiple `hab.site.add_paths` entry_points are processed
+        when not added dynamically."""
+        site = Site(
+            [
+                config_root / "site" / "eps" / "site_add_paths_a.json",
+                config_root / "site" / "eps" / "site_add_paths.json",
+            ]
+        )
+
+        # Check that static and dynamic paths were added in the correct order.
+        # Note: `site_add_paths` ends up adding the `site_add_paths_a.json` path
+        # twice, the first time the path is encountered, all other instances of
+        # that path are discarded.
+        assert len(site.paths) == 4
+        assert site.paths[0].name == "site_add_paths_c.json"
+        assert site.paths[1].name == "site_add_paths_b.json"
+        assert site.paths[2].name == "site_add_paths_a.json"
+        assert site.paths[3].name == "site_add_paths.json"
+
+        # Check which "set" value was resolved by the end. To correctly process
+        # the list returned by the entry_points are processed in reverse order
+        assert site["test_data"] == ["site_add_paths_c.json"]
+
+    def test_site_finalize(self, config_root):
+        """Test that site entry_point `hab.site.finalize` is called.
+
+        This expects that the entry point will raise a `NotImplementedError` with
+        a specific message. This requires that each test has its own site json
+        file enabling that specific entry_point. See `tests/site/eps/README.md`.
+        """
+        with pytest.raises(
+            NotImplementedError,
+            match="hab_test_entry_points.site_finalize called successfully",
+        ):
+            Site(
+                [
+                    config_root / "site" / "eps" / "site_finalize.json",
+                ]
+            )