Add filter configurations at a per-Source granularity

Adds treeless="true" and blobless="true" as attributes to a Combination Source.
During edkrepo clone and checkout, if enableSubmodule="true" on the Combination Source,
the filter setting applies to the submodules as well.

--treeless and treeless="true" have priority over --blobless and blobless="true" if multiple are used.

During edkrepo clone, --treeless and --blobless do not apply to the submodule filter settings.

Related issues:

#239
add filter configurations at a per-Source or per-Remote granularity

#247
clone with both --treeless and --blobless encounter git error

Signed-off-by: Nathaniel Haller <[email protected]>

edkrepo/common/clone_utilities.py

@@ -34,15 +34,12 @@ def generate_clone_cmd(repo_to_clone, workspace_dir, args=None, cache_path=None)
 
     if repo_to_clone.branch:
         clone_cmd_args['target_branch'] = '-b {}'.format(repo_to_clone.branch)
-    
+
     if args:
         try:
-            if args.treeless:
+            if args.treeless or repo_to_clone.treeless:
                 clone_cmd_args['treeless'] = '--filter=tree:0'
-        except AttributeError:
-            pass
-        try:
-            if args.blobless:
+            elif args.blobless or repo_to_clone.blobless:
                 clone_cmd_args['blobless'] = '--filter=blob:none'
         except AttributeError:
             pass

edkrepo_manifest_parser/edk_manifest.py

@@ -27,7 +27,7 @@
 RepoHook = namedtuple('RepoHook', ['source', 'dest_path', 'dest_file', 'remote_url'])
 Combination = namedtuple('Combination', ['name', 'description', 'venv_enable'])
 RepoSource = namedtuple('RepoSource', ['root', 'remote_name', 'remote_url', 'branch', 'commit', 'sparse',
-                                       'enable_submodule', 'tag', 'venv_cfg', 'patch_set'])
+                                       'enable_submodule', 'tag', 'venv_cfg', 'patch_set', 'blobless', 'treeless'])
 PatchSet = namedtuple('PatchSet', ['remote', 'name', 'parent_sha', 'fetch_branch'])
 PatchOperation = namedtuple('PatchOperation',['type', 'file', 'sha', 'source_remote', 'source_branch'])
 SparseSettings = namedtuple('SparseSettings', ['sparse_by_default'])
@@ -1074,6 +1074,14 @@ def __init__(self, element, remotes):
             self.venv_cfg = (element.attrib['venv_cfg'])
         except:
             self.venv_cfg = None
+        try:
+            self.blobless = (element.attrib['blobless'].lower() == 'true')
+        except Exception:
+            self.blobless = False
+        try:
+            self.treeless = (element.attrib['treeless'].lower() == 'true')
+        except Exception:
+            self.treeless = False
 
         if self.branch is None and self.commit is None and self.tag is None and self.patch_set is None:
             raise KeyError(ATTRIBUTE_MISSING_ERROR)
@@ -1084,7 +1092,7 @@ def __init__(self, element, remotes):
     @property
     def tuple(self):
         return RepoSource(self.root, self.remote_name, self.remote_url, self.branch,
-                          self.commit, self.sparse, self.enableSub, self.tag, self.venv_cfg, self.patch_set)
+                          self.commit, self.sparse, self.enableSub, self.tag, self.venv_cfg, self.patch_set, self.blobless, self.treeless)
 
 
 class _SparseSettings():

project_utils/submodule.py

@@ -61,7 +61,7 @@ def _deinit(repo, submodules=None, verbose=False):
     return
 
 
-def _update(repo, submodules=None, verbose=False, recursive=False, cache_path=None):
+def _update(repo, submodules=None, verbose=False, recursive=False, cache_path=None, treeless=False, blobless=False):
     """
     Performs the update of submodules.  This includes the sync and update operations.
 
@@ -84,6 +84,10 @@ def _update(repo, submodules=None, verbose=False, recursive=False, cache_path=No
             cmd.append('--recursive')
         if cache_path is not None:
             cmd.extend(['--reference', cache_path])
+        if treeless:
+            cmd.extend(['--filter=tree:0'])
+        elif blobless:
+            cmd.extend(['--filter=blob:none'])
         output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True)
         ui_functions.display_git_output(output_data, verbose)
     else:
@@ -104,6 +108,10 @@ def _update(repo, submodules=None, verbose=False, recursive=False, cache_path=No
             if cache_path is not None:
                 cmd.extend(['--reference', cache_path])
             cmd.extend(['--', sub.path])
+            if treeless:
+                cmd.extend(['--filter=tree:0'])
+            elif blobless:
+                cmd.extend(['--filter=blob:none'])
             output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True)
             ui_functions.display_git_output(output_data, verbose)
     return
@@ -148,6 +156,33 @@ def _get_submodule_enable(manifest, remote_name, combo):
             return source.enable_submodule
     return False
 
+def _get_blobless_enable(manifest, remote_name, combo):
+    """
+    Determines if blobless filter is enabled for the current repo and combo
+
+    manifest    - Manifest object
+    remote_name - The name of the current remote being processed
+    combo       - The current combo name being processed
+    """
+    repo_sources = manifest.get_repo_sources(combo)
+    for source in repo_sources:
+        if source.remote_name == remote_name:
+            return source.blobless
+    return False
+
+def _get_treeless_enable(manifest, remote_name, combo):
+    """
+    Determines if treeless filter are enabled for the current repo and combo
+
+    manifest    - Manifest object
+    remote_name - The name of the current remote being processed
+    combo       - The current combo name being processed
+    """
+    repo_sources = manifest.get_repo_sources(combo)
+    for source in repo_sources:
+        if source.remote_name == remote_name:
+            return source.treeless
+    return False
 
 def _get_submodule_state(remote_name, start_manifest, start_combo, end_manifest=None, end_combo=None):
     """
@@ -219,7 +254,9 @@ def init_full(workspace, manifest, verbose=False):
                 if args.verbose:
                     print(strings.SUBMOD_EXCEPTION.format(repo_error))
                 continue
-            _update(repo, None, verbose, True)
+            treeless = _get_treeless_enable(manifest, source.remote_name, current_combo)
+            blobless = _get_blobless_enable(manifest, source.remote_name, current_combo)
+            _update(repo, None, verbose, True, treeless=treeless, blobless=blobless)
 
 
 def deinit_submodules(workspace, start_manifest, start_combo,
@@ -307,10 +344,12 @@ def maintain_submodules(workspace, manifest, combo_name, verbose=False, cache_pa
             _init(repo, repo_subs, verbose)
 
         # Perform sync/update
+        treeless = _get_treeless_enable(manifest, source.remote_name, combo_name)
+        blobless = _get_blobless_enable(manifest, source.remote_name, combo_name)
         if len(repo_subs) == 0:
-            _update(repo, None, verbose, cache_path=cache_path)
+            _update(repo, None, verbose, cache_path=cache_path, treeless=treeless, blobless=blobless)
         else:
-            _update(repo, repo_subs, verbose, cache_path=cache_path)
+            _update(repo, repo_subs, verbose, cache_path=cache_path, treeless=treeless, blobless=blobless)
 
 
 if __name__ == '__main__':