Merge pull request #844 from projectsyn/feat/parallel-dependency-fetc…

…hing Fetch component and package dependencies in parallel
projectsyn · Sep 11, 2023 · d70cca6 · d70cca6
2 parents 9077716 + 2ff5d38
commit d70cca6
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 5 deletions.
diff --git a/commodore/cluster.py b/commodore/cluster.py
@@ -196,7 +196,7 @@ def render_target(
 
     classes = [f"params.{inv.bootstrap_target}"]
 
-    for c in components:
+    for c in sorted(components):
         if inv.defaults_file(c).is_file():
             classes.append(f"defaults.{c}")
         else:

diff --git a/commodore/dependency_mgmt/__init__.py b/commodore/dependency_mgmt/__init__.py
@@ -1,5 +1,8 @@
 from __future__ import annotations
 
+import itertools
+from concurrent.futures import ThreadPoolExecutor
+
 import click
 
 from commodore.config import Config
@@ -69,6 +72,8 @@ def fetch_components(cfg: Config):
     cfg.register_component_aliases(component_aliases)
     cspecs = _read_components(cfg, component_names)
     click.secho("Fetching components...", bold=True)
+
+    deps: dict[str, list] = {}
     for cn in component_names:
         cspec = cspecs[cn]
         if cfg.debug:
@@ -86,11 +91,28 @@ def fetch_components(cfg: Config):
                 f"Component {cn} has uncommitted changes. "
                 + "Please specify `--force` to discard them"
             )
+        deps.setdefault(cdep.url, []).append(c)
+    fetch_parallel(fetch_component, cfg, deps.values())
+
+
+def fetch_component(cfg, dependencies):
+    """
+    Fetch all components of a MultiDependency object.
+    """
+    for c in dependencies:
         c.checkout()
         cfg.register_component(c)
         create_component_symlinks(cfg, c)
 
 
+def fetch_parallel(fetch_fun, cfg, to_fetch):
+    """
+    Fetch dependencies in parallel threads with ThreadPoolExecutor.
+    """
+    with ThreadPoolExecutor() as exe:
+        exe.map(fetch_fun, itertools.repeat(cfg), to_fetch)
+
+
 def register_components(cfg: Config):
     """
     Discover components in the inventory, and register them if the
@@ -152,6 +174,7 @@ def fetch_packages(cfg: Config):
     pkgs = _discover_packages(cfg)
     pspecs = _read_packages(cfg, pkgs)
 
+    deps: dict[str, list] = {}
     for p in pkgs:
         pspec = pspecs[p]
         pdep = cfg.register_dependency_repo(pspec.url)
@@ -167,6 +190,15 @@ def fetch_packages(cfg: Config):
                 f"Package {p} has uncommitted changes. "
                 + "Please specify `--force` to discard them"
             )
+        deps.setdefault(pdep.url, []).append((p, pkg))
+    fetch_parallel(fetch_package, cfg, deps.values())
+
+
+def fetch_package(cfg, dependencies):
+    """
+    Fetch all package dependencies of a MultiDependency object.
+    """
+    for p, pkg in dependencies:
         pkg.checkout()
         cfg.register_package(p, pkg)
         create_package_symlink(cfg, p, pkg)

diff --git a/tests/test_target.py b/tests/test_target.py
@@ -53,8 +53,8 @@ def test_render_bootstrap_target(tmp_path: P):
 
     classes = [
         "params.cluster",
-        "defaults.foo",
         "defaults.bar",
+        "defaults.foo",
         "global.commodore",
     ]
     assert target != ""
@@ -83,8 +83,8 @@ def test_render_target(tmp_path: P):
 
     classes = [
         "params.cluster",
-        "defaults.foo",
         "defaults.bar",
+        "defaults.foo",
         "global.commodore",
         "components.foo",
     ]
@@ -115,8 +115,8 @@ def test_render_aliased_target(tmp_path: P):
 
     classes = [
         "params.cluster",
-        "defaults.foo",
         "defaults.bar",
+        "defaults.foo",
         "global.commodore",
         "components.foo",
     ]
@@ -148,8 +148,8 @@ def test_render_aliased_target_with_dash(tmp_path: P):
 
     classes = [
         "params.cluster",
-        "defaults.foo-comp",
         "defaults.bar",
+        "defaults.foo-comp",
         "global.commodore",
         "components.foo-comp",
     ]