diff --git a/README.md b/README.md index 73c81e9..c283c75 100644 --- a/README.md +++ b/README.md @@ -832,6 +832,11 @@ This config would have the URI `project_a/Thug/Animation`. Configs support [min_verbosity](#min_verbosity) [with inheritance](tests/configs/verbosity). +When doing bulk editing of multiple configs, the `hab dump --type all-uris` command +provides you with a easily diff-able json dump of the [freeze](#restoring-resolved-configuration) +for all non-placeholder URI's defined. If a URI errors out when resolving, the +error text is stored instead of the freeze. + #### Config Inheritance When resolving a URI it will find the closest exact match, so if `project_a/Thug` is diff --git a/hab/cli.py b/hab/cli.py index d4eb1cc..4dcbd0a 100644 --- a/hab/cli.py +++ b/hab/cli.py @@ -570,7 +570,7 @@ def env(settings, uri, launch): "--type", "report_type", type=click.Choice( - ["nice", "site", "s", "uris", "u", "versions", "v", "forest", "f"] + ["nice", "site", "s", "uris", "u", "versions", "v", "forest", "f", "all-uris"] ), default="nice", help="Type of report.", @@ -635,6 +635,12 @@ def echo_line(line): truncate=truncate, ): echo_line(line) + elif report_type == "all-uris": + # Combines all non-placeholder URI's into a single json document and display. + # This can be used to compare changes to configs when editing them in bulk. + ret = resolver.freeze_configs() + ret = utils.dumps_json(ret, indent=2) + click.echo(ret) elif report_type == "site": click.echo(resolver.site.dump(verbosity=verbosity)) else: diff --git a/hab/resolver.py b/hab/resolver.py index 2ae682e..7109f17 100644 --- a/hab/resolver.py +++ b/hab/resolver.py @@ -264,6 +264,26 @@ def find_distro(self, requirement): app = self.distros[requirement.name] return app.latest_version(requirement) + def freeze_configs(self): + """Returns a composite dict of the freeze for all URI configs. + + Returns a dict for every non-placeholder URI where the value is the freeze + dict of that URI. If a error is encountered when generating the freeze + the exception subject is stored as a string instead. + """ + out = {} + for node in self.dump_forest(self.configs, attr=None): + if isinstance(node.node, HabBase._placeholder): + continue + uri = node.node.uri + try: + cfg = self.resolve(uri) + except Exception as error: + out[uri] = f"Error resolving {uri}: {error}" + else: + out[uri] = cfg.freeze() + return out + @classmethod def instance(cls, name="main", **kwargs): """Returns a shared Resolver instance for name, initializing it if required. diff --git a/tests/conftest.py b/tests/conftest.py index a6b66b9..0aa1a27 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -175,6 +175,33 @@ def reset_environ(): os.environ.clear() os.environ.update(old_environ) + @staticmethod + def compare_files(generated, check): + """Assert two files are the same with easy to read errors. + + First compares the number of lines for differences, then checks each line + for differences raising an AssertionError on the first difference. + + Args: + generated (pathlib.Path): The file generated for testing. This will + have a newline added to the end to match the pre-commit enforced + "fix end of files" check. + check (pathlib.Path): Compare generated to this check file. It is + normally committed inside the hab/tests folder. + """ + check = check.open().readlines() + cache = generated.open().readlines() + # Add trailing white space to match template file's trailing white space + cache[-1] += "\n" + assert len(cache) == len( + check + ), f"Generated cache does not have the same number of lines: {check}" + + for i in range(len(cache)): + assert ( + cache[i] == check[i] + ), f"Difference on line: {i} between the generated cache and {generated}." + @pytest.fixture def helpers(): diff --git a/tests/resolver_freeze_configs.json b/tests/resolver_freeze_configs.json new file mode 100644 index 0000000..870f526 --- /dev/null +++ b/tests/resolver_freeze_configs.json @@ -0,0 +1,420 @@ +{ + "app/aliased": { + "context": [ + "app" + ], + "name": "aliased", + "versions": [ + "aliased==2.0" + ], + "uri": "app/aliased" + }, + "app/aliased/config": { + "context": [ + "app", + "aliased" + ], + "name": "config", + "versions": [ + "aliased==2.0" + ], + "uri": "app/aliased/config" + }, + "app/aliased/mod": { + "context": [ + "app", + "aliased" + ], + "name": "mod", + "versions": [ + "aliased==2.0", + "aliased_mod==1.0" + ], + "uri": "app/aliased/mod" + }, + "app/aliased/mod/config": { + "context": [ + "app", + "aliased", + "mod" + ], + "name": "config", + "versions": [ + "aliased==2.0", + "aliased_mod==1.0" + ], + "uri": "app/aliased/mod/config" + }, + "app/houdini/a": { + "context": [ + "app", + "houdini" + ], + "name": "a", + "versions": [ + "houdini18.5==18.5.351", + "houdini19.5==19.5.493" + ], + "uri": "app/houdini/a" + }, + "app/houdini/b": { + "context": [ + "app", + "houdini" + ], + "name": "b", + "versions": [ + "houdini19.5==19.5.493", + "houdini18.5==18.5.351" + ], + "uri": "app/houdini/b" + }, + "app/maya": { + "context": [ + "app" + ], + "name": "maya", + "min_verbosity": { + "global": 2, + "hab": 2, + "hab-gui": 1 + }, + "uri": "app/maya" + }, + "app/maya/2020": { + "context": [ + "app", + "maya" + ], + "name": "2020", + "versions": [ + "maya2020==2020.1" + ], + "min_verbosity": { + "global": 2, + "hab": 2, + "hab-gui": 1 + }, + "uri": "app/maya/2020" + }, + "app/maya/2024": { + "context": [ + "app", + "maya" + ], + "name": "2024", + "versions": [ + "maya2024==2024.0" + ], + "min_verbosity": { + "global": 2, + "hab": 2, + "hab-gui": 1 + }, + "uri": "app/maya/2024" + }, + "default": { + "context": [], + "name": "default", + "versions": [ + "the_dcc_plugin_a==1.1", + "the_dcc_plugin_e==1.1", + "the_dcc_plugin_d==1.1", + "the_dcc_plugin_b==1.1", + "the_dcc_plugin_c==1.1", + "maya2020==2020.1", + "houdini18.5==18.5.351" + ], + "uri": "default" + }, + "default/Sc1": { + "context": [ + "default" + ], + "name": "Sc1", + "versions": [ + "the_dcc_plugin_a==1.1", + "the_dcc_plugin_e==1.1", + "the_dcc_plugin_d==1.1", + "the_dcc_plugin_b==1.1", + "the_dcc_plugin_c==1.1", + "maya2020==2020.1", + "houdini18.5==18.5.351" + ], + "uri": "default/Sc1" + }, + "default/Sc11": { + "context": [ + "default" + ], + "name": "Sc11", + "versions": [ + "the_dcc_plugin_a==1.1", + "the_dcc_plugin_e==1.1", + "the_dcc_plugin_d==1.1", + "the_dcc_plugin_b==1.1", + "the_dcc_plugin_c==1.1", + "maya2020==2020.1", + "houdini18.5==18.5.351" + ], + "uri": "default/Sc11" + }, + "not_set": { + "context": [], + "name": "not_set", + "versions": [ + "aliased==2.0", + "maya2020==2020.1" + ], + "uri": "not_set" + }, + "not_set/child": { + "context": [ + "not_set" + ], + "name": "child", + "versions": [ + "aliased==2.0", + "maya2020==2020.1" + ], + "uri": "not_set/child" + }, + "not_set/distros": { + "context": [ + "not_set" + ], + "name": "distros", + "versions": [ + "the_dcc==1.2", + "the_dcc_plugin_a==1.1", + "the_dcc_plugin_e==1.1", + "the_dcc_plugin_d==1.1", + "the_dcc_plugin_b==1.1" + ], + "uri": "not_set/distros" + }, + "not_set/empty_lists": { + "context": [ + "not_set" + ], + "name": "empty_lists", + "uri": "not_set/empty_lists" + }, + "not_set/env1": { + "context": [ + "not_set" + ], + "name": "env1", + "versions": [ + "aliased==2.0", + "maya2020==2020.1" + ], + "uri": "not_set/env1" + }, + "not_set/env_path_hab_uri": "Error resolving not_set/env_path_hab_uri: '\"HAB_URI\" is a reserved environment variable'", + "not_set/env_path_set": "Error resolving not_set/env_path_set: You can not use PATH for the set operation: \"path_variable\"", + "not_set/env_path_unset": "Error resolving not_set/env_path_unset: You can not unset PATH", + "not_set/no_distros": { + "context": [ + "not_set" + ], + "name": "no_distros", + "uri": "not_set/no_distros" + }, + "not_set/no_env": { + "context": [ + "not_set" + ], + "name": "no_env", + "versions": [ + "the_dcc==1.2", + "the_dcc_plugin_a==1.1", + "the_dcc_plugin_e==1.1", + "the_dcc_plugin_d==1.1", + "the_dcc_plugin_b==1.1" + ], + "uri": "not_set/no_env" + }, + "not_set/os": { + "context": [ + "not_set" + ], + "name": "os", + "uri": "not_set/os" + }, + "optional": { + "context": [], + "name": "optional", + "versions": [ + "the_dcc==1.2", + "the_dcc_plugin_a==1.1", + "the_dcc_plugin_e==1.1", + "the_dcc_plugin_d==1.1", + "the_dcc_plugin_b==1.1" + ], + "optional_distros": { + "maya2024": [ + "Adds new aliases" + ], + "the_dcc==1.0": [ + "Specific dcc version. Only choose one at a time." + ], + "the_dcc==1.2": [ + "Specific dcc version. Only choose one at a time." + ], + "the_dcc_plugin_a": [ + "Load an optional plugin by default", + true + ], + "the_dcc_plugin_a==0.9": [ + "Force a specific version of this optinal plugin" + ], + "the_dcc_plugin_b": [ + "Only have a few licenses for this plugin, so opt into loading it" + ] + }, + "uri": "optional" + }, + "optional/child": { + "context": [ + "optional" + ], + "name": "child", + "versions": [ + "the_dcc==1.2", + "the_dcc_plugin_a==1.1", + "the_dcc_plugin_e==1.1", + "the_dcc_plugin_d==1.1", + "the_dcc_plugin_b==1.1" + ], + "optional_distros": { + "the_dcc_plugin_e": [ + "Different optional dependencies for a child URI.", + true + ] + }, + "uri": "optional/child" + }, + "place-holder/child": { + "context": [ + "place-holder" + ], + "name": "child", + "versions": [ + "the_dcc==1.2", + "the_dcc_plugin_a==1.1", + "the_dcc_plugin_e==1.1", + "the_dcc_plugin_d==1.1", + "the_dcc_plugin_b==1.1" + ], + "uri": "place-holder/child" + }, + "place-holder/inherits": { + "context": [ + "place-holder" + ], + "name": "inherits", + "versions": [ + "the_dcc_plugin_a==1.1", + "the_dcc_plugin_e==1.1", + "the_dcc_plugin_d==1.1", + "the_dcc_plugin_b==1.1", + "the_dcc_plugin_c==1.1", + "maya2020==2020.1", + "houdini18.5==18.5.351" + ], + "uri": "place-holder/inherits" + }, + "project_a": { + "context": [], + "name": "project_a", + "versions": [ + "maya2020==2020.1", + "houdini18.5==18.5.351" + ], + "uri": "project_a" + }, + "project_a/Sc001": { + "context": [ + "project_a" + ], + "name": "Sc001", + "versions": [ + "maya2020==2020.0" + ], + "uri": "project_a/Sc001" + }, + "project_a/Sc001/Animation": { + "context": [ + "project_a", + "Sc001" + ], + "name": "Animation", + "versions": [ + "maya2020==2020.1" + ], + "uri": "project_a/Sc001/Animation" + }, + "project_a/Sc001/Rigging": { + "context": [ + "project_a", + "Sc001" + ], + "name": "Rigging", + "versions": [ + "maya2020==2020.1" + ], + "uri": "project_a/Sc001/Rigging" + }, + "verbosity": { + "context": [], + "name": "verbosity", + "min_verbosity": { + "global": 2, + "hab-gui": 1 + }, + "uri": "verbosity" + }, + "verbosity/hidden": { + "context": [ + "verbosity" + ], + "name": "hidden", + "min_verbosity": { + "global": 3, + "hab-gui": 2 + }, + "uri": "verbosity/hidden" + }, + "verbosity/inherit": { + "context": [ + "verbosity" + ], + "name": "inherit", + "versions": [ + "aliased_verbosity==1.0" + ], + "min_verbosity": { + "global": 2, + "hab-gui": 1 + }, + "uri": "verbosity/inherit" + }, + "verbosity/inherit-no": { + "context": [ + "verbosity" + ], + "name": "inherit-no", + "uri": "verbosity/inherit-no" + }, + "verbosity/inherit-override": { + "context": [ + "verbosity" + ], + "name": "inherit-override", + "min_verbosity": { + "global": 1 + }, + "uri": "verbosity/inherit-override" + } +} diff --git a/tests/test_cache.py b/tests/test_cache.py index 3a39c98..27dbd91 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -42,23 +42,12 @@ def test_site_cache_path(config_root, uncached_resolver, tmpdir): assert site.cache.cache_template == ".{stem}.hab_cache" -def test_save_cache(config_root, tmpdir, habcached_resolver): +def test_save_cache(config_root, tmpdir, habcached_resolver, helpers): # Check that the habcache file generated the expected output text # Note: This file will need updated as the test configuration is updated check_path = config_root / "site_main_check.habcache" cache_file = habcached_resolver._test_cache_file - check = check_path.open().readlines() - cache = cache_file.open().readlines() - # Add trailing white space to match template file's trailing white space - cache[-1] += "\n" - assert len(cache) == len( - check - ), f"Generated cache does not have the same number of lines: {cache_file}" - - for i in range(len(cache)): - assert ( - cache[i] == check[i] - ), f"Difference on line: {i} between the generated cache and {check_path}." + helpers.compare_files(cache_file, check_path) def test_load_cache(config_root, uncached_resolver, habcached_site_file): diff --git a/tests/test_freeze.py b/tests/test_freeze.py index 8249297..242f372 100644 --- a/tests/test_freeze.py +++ b/tests/test_freeze.py @@ -230,3 +230,32 @@ def test_encode_freeze(config_root, resolver): # If version is passed, site is ignored version1 = utils.encode_freeze(freeze, version=2, site=site) assert version1 == checks["version2"] + + +def test_resolver_freeze_configs(tmpdir, config_root, uncached_resolver, helpers): + """Test `Resolver.freeze_configs`. + + This method generates the frozen config for all non-placeholder URI's hab + finds. This makes it fairly easy to diff bulk config changes as json. + + It checks the output against `tests/resolver_freeze_configs.json`. For testing + simplicity, this file has had its aliases and environment sections removed. + """ + result = uncached_resolver.freeze_configs() + # Simplify the test by removing dynamic data containing paths. Other tests + # verify that a specific URI can be frozen successfully. This test verifies + # that freeze_configs generates a consistent output for all URI's. + for data in result.values(): + if isinstance(data, str): + continue + if "aliases" in data: + del data["aliases"] + if "environment" in data: + del data["environment"] + + check_file = config_root / "resolver_freeze_configs.json" + result_file = tmpdir / "result.json" + with result_file.open("w") as fle: + txt = utils.dumps_json(result, indent=4) + fle.write(txt) + helpers.compare_files(result_file, check_file)