From 5a62df6e625fd9e2992dc6bbe202793b606894f5 Mon Sep 17 00:00:00 2001 From: haasad Date: Fri, 8 Sep 2023 16:27:32 +0200 Subject: [PATCH 1/4] Fetch component dependencies in parallel --- commodore/dependency_mgmt/__init__.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/commodore/dependency_mgmt/__init__.py b/commodore/dependency_mgmt/__init__.py index 7acfcbb2..d74582ac 100644 --- a/commodore/dependency_mgmt/__init__.py +++ b/commodore/dependency_mgmt/__init__.py @@ -1,5 +1,8 @@ from __future__ import annotations +import itertools +from concurrent.futures import ThreadPoolExecutor + import click from commodore.config import Config @@ -69,6 +72,8 @@ def fetch_components(cfg: Config): cfg.register_component_aliases(component_aliases) cspecs = _read_components(cfg, component_names) click.secho("Fetching components...", bold=True) + + deps: dict[str, list] = {} for cn in component_names: cspec = cspecs[cn] if cfg.debug: @@ -86,11 +91,28 @@ def fetch_components(cfg: Config): f"Component {cn} has uncommitted changes. " + "Please specify `--force` to discard them" ) + deps.setdefault(cdep.url, []).append(c) + fetch_parallel(fetch_component, cfg, deps.values()) + + +def fetch_component(cfg, dependencies): + """ + Fetch all components of a MultiDependency object. + """ + for c in dependencies: c.checkout() cfg.register_component(c) create_component_symlinks(cfg, c) +def fetch_parallel(fetch_fun, cfg, to_fetch): + """ + Fetch dependencies in parallel threads with ThreadPoolExecutor. + """ + with ThreadPoolExecutor() as exe: + exe.map(fetch_fun, itertools.repeat(cfg), to_fetch) + + def register_components(cfg: Config): """ Discover components in the inventory, and register them if the From 10bb82a563a3fa91e51373509ea7481d31c61bf2 Mon Sep 17 00:00:00 2001 From: haasad Date: Fri, 8 Sep 2023 17:16:31 +0200 Subject: [PATCH 2/4] Fetch package dependencies in parallel --- commodore/dependency_mgmt/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/commodore/dependency_mgmt/__init__.py b/commodore/dependency_mgmt/__init__.py index d74582ac..03da1b4c 100644 --- a/commodore/dependency_mgmt/__init__.py +++ b/commodore/dependency_mgmt/__init__.py @@ -174,6 +174,7 @@ def fetch_packages(cfg: Config): pkgs = _discover_packages(cfg) pspecs = _read_packages(cfg, pkgs) + deps: dict[str, list] = {} for p in pkgs: pspec = pspecs[p] pdep = cfg.register_dependency_repo(pspec.url) @@ -189,6 +190,15 @@ def fetch_packages(cfg: Config): f"Package {p} has uncommitted changes. " + "Please specify `--force` to discard them" ) + deps.setdefault(pdep.url, []).append((p, pkg)) + fetch_parallel(fetch_package, cfg, deps.values()) + + +def fetch_package(cfg, dependencies): + """ + Fetch all package dependencies of a MultiDependency object. + """ + for p, pkg in dependencies: pkg.checkout() cfg.register_package(p, pkg) create_package_symlink(cfg, p, pkg) From 50925bfcb59a1934a9c73162b056ebbfe224052f Mon Sep 17 00:00:00 2001 From: haasad Date: Mon, 11 Sep 2023 09:58:12 +0200 Subject: [PATCH 3/4] Ensure componentes are sorted With multithreading the order is not deterministic anymore which breaks the integration tests --- commodore/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commodore/cluster.py b/commodore/cluster.py index 5d3fbb13..35cab699 100644 --- a/commodore/cluster.py +++ b/commodore/cluster.py @@ -196,7 +196,7 @@ def render_target( classes = [f"params.{inv.bootstrap_target}"] - for c in components: + for c in sorted(components): if inv.defaults_file(c).is_file(): classes.append(f"defaults.{c}") else: From 2ff5d38e990604f63c89d3d8ced6a60b1103529f Mon Sep 17 00:00:00 2001 From: haasad Date: Mon, 11 Sep 2023 10:33:27 +0200 Subject: [PATCH 4/4] Classes are now explicitly sorted, test need to reflect that --- tests/test_target.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_target.py b/tests/test_target.py index d757d43e..40074d18 100644 --- a/tests/test_target.py +++ b/tests/test_target.py @@ -53,8 +53,8 @@ def test_render_bootstrap_target(tmp_path: P): classes = [ "params.cluster", - "defaults.foo", "defaults.bar", + "defaults.foo", "global.commodore", ] assert target != "" @@ -83,8 +83,8 @@ def test_render_target(tmp_path: P): classes = [ "params.cluster", - "defaults.foo", "defaults.bar", + "defaults.foo", "global.commodore", "components.foo", ] @@ -115,8 +115,8 @@ def test_render_aliased_target(tmp_path: P): classes = [ "params.cluster", - "defaults.foo", "defaults.bar", + "defaults.foo", "global.commodore", "components.foo", ] @@ -148,8 +148,8 @@ def test_render_aliased_target_with_dash(tmp_path: P): classes = [ "params.cluster", - "defaults.foo-comp", "defaults.bar", + "defaults.foo-comp", "global.commodore", "components.foo-comp", ]