Add filter configurations at a per-Source granularity

Adds treeless="true" and blobless="true" as attributes to a Combination Source.
Add arg --full to disable repo attributes.

--treeless, --blobless, and --full args will disable the per-Source attributes.
Allow up to one of --treeless, --blobless and --full args to be used per edkrepo clone.

Related issues:

tianocore#239
add filter configurations at a per-Source or per-Remote granularity

tianocore#247
clone with both --treeless and --blobless encounter git error

Signed-off-by: Nathaniel Haller <[email protected]>

edkrepo/commands/arguments/clone_args.py

@@ -23,11 +23,16 @@
 TREELESS_HELP = ('Creates a partial "treeless" clone; all reachable commits will be downloaded with additional blobs and trees being '
                  'downloaded on demand by future Git operations as needed.\n'
                  'Treeless clones result in significantly faster initial clone times and minimize the amount of content downloaded.\n'
-                 'Workspaces created with this option are best used for one time workspaces that will be discarded.')
+                 'Workspaces created with this option are best used for one time workspaces that will be discarded.\n'
+                 'Any project manifest partial clone settings are ignored.')
 BLOBLESS_HELP = ('Creates a partial "blobless" clone; all reachable commits and trees will be downloaded with additional blobs being '
                  'downloaded on demand by future Git operations as needed.\n'
                  'Blobless clones result in significantly faster initial clone times and minimize the amount of content downloaded.\n'
-                 'Workspaces created with this option are best used for persistent development environments')
+                 'Workspaces created with this option are best used for persistent development environments.\n'
+                 'Any project manifest partial clone settings are ignored.')
+FULL_HELP = ('Creates a "full" clone; all reachable commits, trees, and blobs will be downloaded.\n'
+                 'Full clones have the greatest initial clone times and amount of content downloaded.\n'
+                 'Any project manifest partial clone settings are ignored.')
 SINGLE_BRANCH_HELP = ('Clone only the history leading to the tip of a single branch for each repository in the workspace.\n'
                       'The branch is determined by the default combination or by the Combination parameter.')
 NO_TAGS_HELP = ('Skips download of tags and updates config settings to ensure that future pull and fetch operations do not follow tags.\n'

edkrepo/commands/clone_command.py

@@ -76,6 +76,10 @@ def get_metadata(self):
                      'positional': False,
                      'required': False,
                      'help-text': arguments.BLOBLESS_HELP})
+        args.append({'name': 'full',
+                     'positional': False,
+                     'required': False,
+                     'help-text': arguments.FULL_HELP})
         args.append(({'name': 'single-branch',
                      'positional': False,
                      'required': False,

edkrepo/common/clone_utilities.py

@@ -16,6 +16,7 @@
 import edkrepo.common.edkrepo_exception as edkrepo_exception
 import edkrepo.common.humble as humble
 import edkrepo.common.ui_functions as ui_functions
+import edkrepo.commands.humble.clone_humble as humble
 
 def generate_clone_cmd(repo_to_clone, workspace_dir, args=None, cache_path=None):
     '''Generates and returns a string representing a git clone command which can be passed to subprocess for execution.
@@ -31,19 +32,28 @@ def generate_clone_cmd(repo_to_clone, workspace_dir, args=None, cache_path=None)
     base_clone_cmd_cache = 'git clone {} {} --reference-if-able {} --progress'.format(repo_to_clone.remote_url, local_repo_path, cache_path)
     clone_arg_string = None
     clone_cmd_args = {}
+    active_filters = []
 
     if repo_to_clone.branch:
         clone_cmd_args['target_branch'] = '-b {}'.format(repo_to_clone.branch)
-    
+
     if args:
         try:
             if args.treeless:
                 clone_cmd_args['treeless'] = '--filter=tree:0'
+                active_filters.append('treeless')
         except AttributeError:
             pass
         try:
             if args.blobless:
                 clone_cmd_args['blobless'] = '--filter=blob:none'
+                active_filters.append('blobless')
+        except AttributeError:
+            pass
+        try:
+            if args.full:
+                clone_cmd_args['full'] = ''
+                active_filters.append('full')
         except AttributeError:
             pass
         try:
@@ -57,6 +67,25 @@ def generate_clone_cmd(repo_to_clone, workspace_dir, args=None, cache_path=None)
         except AttributeError:
             pass
 
+    try:
+        if repo_to_clone.treeless:
+            clone_cmd_args['treeless repo'] = '--filter=tree:0'
+    except AttributeError:
+        pass
+    try:
+        if repo_to_clone.blobless:
+            clone_cmd_args['blobless repo'] = '--filter=blob:none'
+    except AttributeError:
+        pass
+
+    if len(active_filters) > 1:
+        raise edkrepo_exception.EdkrepoInvalidConfigOptionException(humble.CONFLICTING_PARTIAL_CLONE)
+    elif len(active_filters) == 1:
+        clone_cmd_args['treeless repo'] = ''
+        clone_cmd_args['blobless repo'] = ''
+    elif 'treeless repo' in clone_cmd_args and 'blobless repo' in clone_cmd_args:
+        raise edkrepo_exception.EdkrepoInvalidConfigOptionException(humble.CONFLICTING_PARTIAL_CLONE)
+
     if clone_cmd_args:
         clone_arg_string = ' '.join([clone_cmd_args[arg] for arg in clone_cmd_args.keys()])
 

edkrepo_manifest_parser/edk_manifest.py

@@ -27,7 +27,7 @@
 RepoHook = namedtuple('RepoHook', ['source', 'dest_path', 'dest_file', 'remote_url'])
 Combination = namedtuple('Combination', ['name', 'description', 'venv_enable'])
 RepoSource = namedtuple('RepoSource', ['root', 'remote_name', 'remote_url', 'branch', 'commit', 'sparse',
-                                       'enable_submodule', 'tag', 'venv_cfg', 'patch_set'])
+                                       'enable_submodule', 'tag', 'venv_cfg', 'patch_set', 'blobless', 'treeless'])
 PatchSet = namedtuple('PatchSet', ['remote', 'name', 'parent_sha', 'fetch_branch'])
 PatchOperation = namedtuple('PatchOperation',['type', 'file', 'sha', 'source_remote', 'source_branch'])
 SparseSettings = namedtuple('SparseSettings', ['sparse_by_default'])
@@ -1074,6 +1074,14 @@ def __init__(self, element, remotes):
             self.venv_cfg = (element.attrib['venv_cfg'])
         except:
             self.venv_cfg = None
+        try:
+            self.blobless = (element.attrib['blobless'].lower() == 'true')
+        except Exception:
+            self.blobless = False
+        try:
+            self.treeless = (element.attrib['treeless'].lower() == 'true')
+        except Exception:
+            self.treeless = False
 
         if self.branch is None and self.commit is None and self.tag is None and self.patch_set is None:
             raise KeyError(ATTRIBUTE_MISSING_ERROR)
@@ -1084,7 +1092,7 @@ def __init__(self, element, remotes):
     @property
     def tuple(self):
         return RepoSource(self.root, self.remote_name, self.remote_url, self.branch,
-                          self.commit, self.sparse, self.enableSub, self.tag, self.venv_cfg, self.patch_set)
+                          self.commit, self.sparse, self.enableSub, self.tag, self.venv_cfg, self.patch_set, self.blobless, self.treeless)
 
 
 class _SparseSettings():