diff --git a/README.md b/README.md index ec275f7..1b9bc05 100644 --- a/README.md +++ b/README.md @@ -769,21 +769,58 @@ and `Sc11`. The URI of `not_a_project/Sc101` would end up using `default/Sc1`. T `not_a_project/Sc110` would use `default/Sc11`. The URI `not_a_project/Sc200` would use `default`. -### Variable Formatting +### Str Formatting -The configuration environment variables and aliases can be formatted using str.format syntax. +The configuration environment variables and aliases can be formatted using +`str.format`. Hab extends the python [Format Specification Mini-Language](https://docs.python.org/3/library/string.html#formatspec) +with these extra features: +* `{ANYTHING!e}`: `!e` is a special conversion flag for Environment variables. This will +be replaced with the correct shell environment variable. For bash it becomes `$ANYTHING`, +in power shell `$env:ANYTHING`, and in command prompt `%ANYTHING%`. `ANYTHING` is the name +of the environment variable. -Currently supported variables: +#### Hab specific variables + +Hab defines some specific variables. These are used when parsing an individual json file: * `{relative_root}`: The directory name of the .json config file. Think of this as the relative path `.` when using the command line, but this is a clear indication that it needs to be replaced with the dirname and not left alone. -* `{ANYTHING!e}`: `!e` is a special conversion flag for Environment variables. This will -be replaced with the correct shell environment variable. For bash it becomes `$ANYTHING`, -in power shell `$env:ANYTHING`, and in command prompt `%ANYTHING%`. ANYTHING is the name -of the environment variable. * `{;}`: This is replaced with the path separator for the shell. Ie `:` for bash, and `;` on windows(including bash). +#### Custom variables + +You can define custom variables in a json file by defining a "variables" dictionary. +You can use these keys in other parts of that file. An `ReservedVariableNameError` +will be raised if you try to replace hab specific variables like `relative_root` and `;`. + +```json5 +{ + "name": "maya2024", + "variables": { + "maya_root_linux": "/usr/autodesk/maya2024/bin", + "maya_root_windows": "C:/Program Files/Autodesk/Maya2024/bin" + }, + "aliases": { + "linux": [ + ["maya", {"cmd": "{maya_root_linux}/maya"}] + ], + "windows": [ + ["maya", {"cmd": "{maya_root_windows}\\maya.exe"}] + ] + } +} +``` + +The [maya2024](tests/distros/maya2024/2024.0/.hab.json) testing example shows a +way to centralize the path to the Maya bin directory. This way automated tools +can easily change the install directory of maya if it needs to be installed +into a custom location on specific workstations, or remote computers. + +Note: Using hard coded paths like `maya_root_windows` should be avoided unless +you really can't use `relative_root`. `relative_root` is more portable across +workstation setups as it doesn't require any modifications when the path changes. + ### Min_Verbosity [Config](#config) and [aliases](#hiding-aliases) can be hidden depending on the diff --git a/hab/errors.py b/hab/errors.py index e292686..838e1e2 100644 --- a/hab/errors.py +++ b/hab/errors.py @@ -52,3 +52,20 @@ def __str__(self): if self.error: ret = f"[{type(self.error).__name__}] {self.error}\n{ret}" return ret + + +class ReservedVariableNameError(HabError): + """Raised if a custom variable uses a reserved variable name.""" + + _reserved_variable_names = {"relative_root", ";"} + """A set of variable names hab reserved for hab use and should not be defined + by custom variables.""" + + def __init__(self, invalid, filename): + self.filename = filename + msg = ( + f"{', '.join(sorted(invalid))!r} are reserved variable name(s) for " + "hab and can not be used in the variables section. " + f"Filename: '{self.filename}'" + ) + super().__init__(msg) diff --git a/hab/parsers/hab_base.py b/hab/parsers/hab_base.py index 981ba7f..69241b0 100644 --- a/hab/parsers/hab_base.py +++ b/hab/parsers/hab_base.py @@ -9,7 +9,7 @@ from packaging.version import Version from .. import NotSet, utils -from ..errors import DuplicateJsonError, HabError +from ..errors import DuplicateJsonError, HabError, ReservedVariableNameError from ..formatter import Formatter from ..site import MergeDict from ..solvers import Solver @@ -48,6 +48,7 @@ def __init__(self, forest, resolver, filename=None, parent=None, root_paths=None self._filename = None self._dirname = None self._distros = NotSet + self._variables = NotSet self._uri = NotSet self.parent = parent self.root_paths = set() @@ -472,9 +473,14 @@ def format_environment_value(self, value, ext=None, platform=None): # Just return boolean values, no need to format return value - # Expand and format any variables like "relative_root" using the current - # platform for paths. - kwargs = dict(relative_root=utils.path_forward_slash(self.dirname)) + # Include custom variables in the format dictionary. + kwargs = {} + if self.variables: + kwargs.update(self.variables) + + # Custom variables do not override hab variables, ensure they are set + # to the correct value. + kwargs["relative_root"] = utils.path_forward_slash(self.dirname) ret = Formatter(ext).format(value, **kwargs) # Use site.the platform_path_maps to convert the result to the target platform @@ -553,6 +559,8 @@ def load(self, filename, data=None): # Check for NotSet so sub-classes can set values before calling super if self.name is NotSet: self.name = data["name"] + if self.variables is NotSet: + self.variables = data.get("variables", NotSet) if "version" in data and self.version is NotSet: self.version = data.get("version") if self.distros is NotSet: @@ -706,6 +714,25 @@ def _apply(data): if alias_name: _apply(self.aliases[alias_name].get("environment", {})) + @hab_property(verbosity=3) + def variables(self): + """A configurable dict of reusable key/value pairs to use when formatting + text strings in the rest of the json file. This value is stored in the + `variables` dictionary of the json file. + """ + return self._variables + + @variables.setter + def variables(self, variables): + # Raise an exception if a reserved variable name is used. + if variables and isinstance(variables, dict): + invalid = ReservedVariableNameError._reserved_variable_names.intersection( + variables + ) + if invalid: + raise ReservedVariableNameError(invalid, self.filename) + self._variables = variables + @property def version(self): """A `packaging.version.Version` representing the version of this object.""" diff --git a/tests/configs/app/app_maya_2024.json b/tests/configs/app/app_maya_2024.json new file mode 100644 index 0000000..635d3e6 --- /dev/null +++ b/tests/configs/app/app_maya_2024.json @@ -0,0 +1,8 @@ +{ + "name": "2024", + "context": ["app", "maya"], + "inherits": true, + "distros": [ + "maya2024" + ] +} diff --git a/tests/configs/project_a/project_a.json b/tests/configs/project_a/project_a.json index b643a42..af0ca7c 100644 --- a/tests/configs/project_a/project_a.json +++ b/tests/configs/project_a/project_a.json @@ -5,5 +5,32 @@ "distros": [ "maya2020", "houdini18.5" - ] + ], + "environment": { + "os_specific": true, + "linux": { + "set": { + "OCIO": "{mount_linux}/project_a/cfg/ocio/v0001/config.ocio", + "HOUDINI_OTLSCAN_PATH": [ + "{mount_linux}/project_a/cfg/hdas", + "{mount_linux}/_shared/cfg/hdas", + "&" + ] + } + }, + "windows": { + "set": { + "OCIO": "{mount_windows}/project_a/cfg/ocio/v0001/config.ocio", + "HOUDINI_OTLSCAN_PATH": [ + "{mount_windows}/project_a/cfg/hdas", + "{mount_windows}/_shared/cfg/hdas", + "&" + ] + } + } + }, + "variables": { + "mount_linux": "/blur/g", + "mount_windows": "G:" + } } diff --git a/tests/distros/maya2024/2024.0/.hab.json b/tests/distros/maya2024/2024.0/.hab.json new file mode 100644 index 0000000..183eee8 --- /dev/null +++ b/tests/distros/maya2024/2024.0/.hab.json @@ -0,0 +1,79 @@ +{ + "name": "maya2024", + "variables": { + "maya_root_linux": "/usr/autodesk/maya2024/bin", + "maya_root_windows": "C:/Program Files/Autodesk/Maya2024/bin" + }, + "aliases": { + "windows": [ + [ + "maya", { + "cmd": "{maya_root_windows}/maya.exe" + } + ], + [ + "mayapy", { + "cmd": "{maya_root_windows}/mayapy.exe", + "min_verbosity": {"global": 2} + } + ], + [ + "maya24", { + "cmd": "{maya_root_windows}/maya.exe", + "min_verbosity": {"global": 1} + } + ], + [ + "mayapy24", { + "cmd": "{maya_root_windows}/mayapy.exe", + "min_verbosity": {"global": 2} + } + ], + [ + "pip", { + "cmd": [ + "{maya_root_windows}/mayapy.exe", + "-m", + "pip" + ], + "min_verbosity": {"global": 2} + } + ] + ], + "linux": [ + [ + "maya", { + "cmd": "{maya_root_linux}/maya" + } + ], + [ + "mayapy", { + "cmd": "{maya_root_linux}/mayapy", + "min_verbosity": {"global": 2} + } + ], + [ + "maya24", { + "cmd": "{maya_root_linux}/maya", + "min_verbosity": {"global": 1} + } + ], + [ + "mayapy24", { + "cmd": "{maya_root_linux}/mayapy", + "min_verbosity": {"global": 2} + } + ], + [ + "pip", { + "cmd": [ + "{maya_root_linux}/mayapy", + "-m", + "pip" + ], + "min_verbosity": {"global": 2} + } + ] + ] + } +} diff --git a/tests/site_main_check.habcache b/tests/site_main_check.habcache index 2f8e17d..a1de2a3 100644 --- a/tests/site_main_check.habcache +++ b/tests/site_main_check.habcache @@ -133,6 +133,17 @@ "maya2020" ] }, + "{config-root}/configs/app/app_maya_2024.json": { + "name": "2024", + "context": [ + "app", + "maya" + ], + "inherits": true, + "distros": [ + "maya2024" + ] + }, "{config-root}/configs/default/default.json": { "name": "default", "context": [], @@ -384,7 +395,34 @@ "distros": [ "maya2020", "houdini18.5" - ] + ], + "environment": { + "os_specific": true, + "linux": { + "set": { + "OCIO": "{mount_linux}/project_a/cfg/ocio/v0001/config.ocio", + "HOUDINI_OTLSCAN_PATH": [ + "{mount_linux}/project_a/cfg/hdas", + "{mount_linux}/_shared/cfg/hdas", + "&" + ] + } + }, + "windows": { + "set": { + "OCIO": "{mount_windows}/project_a/cfg/ocio/v0001/config.ocio", + "HOUDINI_OTLSCAN_PATH": [ + "{mount_windows}/project_a/cfg/hdas", + "{mount_windows}/_shared/cfg/hdas", + "&" + ] + } + } + }, + "variables": { + "mount_linux": "/blur/g", + "mount_windows": "G:" + } }, "{config-root}/configs/project_a/project_a_Sc001.json": { "name": "Sc001", @@ -1215,6 +1253,112 @@ ] } }, + "{config-root}/distros/maya2024/2024.0/.hab.json": { + "name": "maya2024", + "variables": { + "maya_root_linux": "/usr/autodesk/maya2024/bin", + "maya_root_windows": "C:/Program Files/Autodesk/Maya2024/bin" + }, + "aliases": { + "windows": [ + [ + "maya", + { + "cmd": "{maya_root_windows}/maya.exe" + } + ], + [ + "mayapy", + { + "cmd": "{maya_root_windows}/mayapy.exe", + "min_verbosity": { + "global": 2 + } + } + ], + [ + "maya24", + { + "cmd": "{maya_root_windows}/maya.exe", + "min_verbosity": { + "global": 1 + } + } + ], + [ + "mayapy24", + { + "cmd": "{maya_root_windows}/mayapy.exe", + "min_verbosity": { + "global": 2 + } + } + ], + [ + "pip", + { + "cmd": [ + "{maya_root_windows}/mayapy.exe", + "-m", + "pip" + ], + "min_verbosity": { + "global": 2 + } + } + ] + ], + "linux": [ + [ + "maya", + { + "cmd": "{maya_root_linux}/maya" + } + ], + [ + "mayapy", + { + "cmd": "{maya_root_linux}/mayapy", + "min_verbosity": { + "global": 2 + } + } + ], + [ + "maya24", + { + "cmd": "{maya_root_linux}/maya", + "min_verbosity": { + "global": 1 + } + } + ], + [ + "mayapy24", + { + "cmd": "{maya_root_linux}/mayapy", + "min_verbosity": { + "global": 2 + } + } + ], + [ + "pip", + { + "cmd": [ + "{maya_root_linux}/mayapy", + "-m", + "pip" + ], + "min_verbosity": { + "global": 2 + } + } + ] + ] + }, + "version": "2024.0" + }, "{config-root}/distros/the_dcc/1.0/.hab.json": { "name": "the_dcc", "environment": {}, diff --git a/tests/test_parsing.py b/tests/test_parsing.py index b5128aa..487c351 100644 --- a/tests/test_parsing.py +++ b/tests/test_parsing.py @@ -10,8 +10,13 @@ from packaging.version import Version from hab import NotSet, utils -from hab.errors import DuplicateJsonError, InvalidVersionError, _IgnoredVersionError -from hab.parsers import Config, DistroVersion +from hab.errors import ( + DuplicateJsonError, + InvalidVersionError, + ReservedVariableNameError, + _IgnoredVersionError, +) +from hab.parsers import Config, DistroVersion, FlatConfig def test_distro_parse(config_root, resolver): @@ -214,6 +219,7 @@ def test_metaclass(): "filename", "min_verbosity", "name", + "variables", "version", ] ) @@ -229,6 +235,7 @@ def test_metaclass(): "inherits", "name", "uri", + "variables", ] ) @@ -987,3 +994,102 @@ def check_freeze(env): del check["ALIASED_GLOBAL_D"] check_freeze(env) assert env == check + + +class TestCustomVariables: + def test_distro(self, uncached_resolver): + """Test that a distro processes valid custom variables correctly.""" + distro = uncached_resolver.distros["maya2024"].latest_version("maya2024") + assert distro.name == "maya2024==2024.0" + + # Check that the custom variables are assigned + check = { + "maya_root_linux": "/usr/autodesk/maya2024/bin", + "maya_root_windows": "C:/Program Files/Autodesk/Maya2024/bin", + } + assert distro.variables == check + + # Check that the aliases have not had their custom vars replaced + alias = distro.aliases["windows"][0] + assert alias[0] == "maya" + assert alias[1]["cmd"] == "{maya_root_windows}/maya.exe" + alias = distro.aliases["linux"][0] + assert alias[0] == "maya" + assert alias[1]["cmd"] == "{maya_root_linux}/maya" + + # Check that the custom variables are replaced when formatting + formatted = distro.format_environment_value(distro.aliases) + + alias = formatted["windows"][0] + assert alias[0] == "maya" + assert alias[1]["cmd"] == f"{check['maya_root_windows']}/maya.exe" + alias = formatted["linux"][0] + assert alias[0] == "maya" + assert alias[1]["cmd"] == f"{check['maya_root_linux']}/maya" + + @pytest.mark.parametrize("config_class", ("Config", "FlatConfig")) + def test_config(self, uncached_resolver, config_class): + """Test that a config processes valid custom variables correctly.""" + if config_class == "Config": + cfg = uncached_resolver.closest_config("project_a") + assert type(cfg) == Config + else: + cfg = uncached_resolver.resolve("project_a") + assert type(cfg) == FlatConfig + + check = { + "mount_linux": "/blur/g", + "mount_windows": "G:", + } + assert cfg.variables == check + + cfg.environment + env = cfg.frozen_data["environment"] + # Check the mount_linux variable was replaced correctly + assert env["linux"]["HOUDINI_OTLSCAN_PATH"] == [ + "/blur/g/project_a/cfg/hdas", + "/blur/g/_shared/cfg/hdas", + "&", + ] + assert env["linux"]["OCIO"] == ["/blur/g/project_a/cfg/ocio/v0001/config.ocio"] + + # Check the mount_windows variable was replaced correctly + assert env["windows"]["HOUDINI_OTLSCAN_PATH"] == [ + "G:/project_a/cfg/hdas", + "G:/_shared/cfg/hdas", + "&", + ] + assert env["windows"]["OCIO"] == ["G:/project_a/cfg/ocio/v0001/config.ocio"] + + @pytest.mark.parametrize( + "variables,invalid", + ( + ({"relative_root": "Not Valid"}, "relative_root"), + ({"relative_root": "Not Valid", ";": "Not Valid"}, ";, relative_root"), + ({"valid": "Valid", ";": "Not Valid"}, ";"), + ), + ) + def test_reserved(self, uncached_resolver, variables, invalid, tmpdir): + """Test that using a reserved variable raises an exception.""" + + # Create a test distro file using the given reserved variables + template = { + "name": "maya2024", + "version": "2024.99", + "variables": variables, + } + distro_file = tmpdir / "maya2024" / ".hab.json" + Path(distro_file).parent.mkdir() + with distro_file.open("w") as fle: + json.dump(template, fle, indent=4, cls=utils.HabJsonEncoder) + + # Add the test distro to hab's distro search. We don't need to call + # `clear_caches` because distros haven't been resolved yet. + uncached_resolver.distro_paths.append(Path(tmpdir)) + + # When distros are resolved, an exception should be raised + with pytest.raises( + ReservedVariableNameError, + match=rf"'{invalid}' are reserved variable name\(s\) for hab", + ): + uncached_resolver.distros diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 4d22c35..65997e6 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -117,6 +117,7 @@ class TestDumpForest: " app/houdini/b", " app/maya", " app/maya/2020", + " app/maya/2024", "default", " default/Sc1", " default/Sc11", @@ -166,6 +167,7 @@ def test_uris_target_default(self, resolver): result = list(resolver.dump_forest(resolver.configs)) check.remove(" app/maya") check.remove(" app/maya/2020") + check.remove(" app/maya/2024") check.remove("verbosity") check.remove(" verbosity/inherit") assert result == check @@ -213,6 +215,7 @@ def test_uris_target_hab_gui(self, resolver): result = list(resolver.dump_forest(resolver.configs)) check.remove(" app/maya") check.remove(" app/maya/2020") + check.remove(" app/maya/2024") check.remove("verbosity") check.remove(" verbosity/inherit") check.remove(" verbosity/inherit-override") @@ -237,6 +240,8 @@ def test_distros(self, resolver): "maya2020", " maya2020==2020.0", " maya2020==2020.1", + "maya2024", + " maya2024==2024.0", "the_dcc", " the_dcc==1.0", " the_dcc==1.1", @@ -283,6 +288,8 @@ def test_distros_truncate(self, resolver): "maya2020", " maya2020==2020.0", " maya2020==2020.1", + "maya2024", + " maya2024==2024.0", "the_dcc", " the_dcc==1.0", " ...",