From a6d021a96b126ddf0d1b7c34d0c4d5c4d571d37b Mon Sep 17 00:00:00 2001 From: Ashley E Desimone Date: Mon, 3 Oct 2022 10:10:48 -0700 Subject: [PATCH 01/23] EdkRepo: Update Comb to Combo in Manifest String Titles Update the spelling of comb to combo in the names of strings defined in edkrepo_manifest.py Signed-off-by: Harsh Vora Signed-off-by: Ashley DeSimone Co-authored-by: Ashley Desimone Co-authored-by: Harsh Vora Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo_manifest_parser/edk_manifest.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/edkrepo_manifest_parser/edk_manifest.py b/edkrepo_manifest_parser/edk_manifest.py index 0bee37e..726e7f9 100644 --- a/edkrepo_manifest_parser/edk_manifest.py +++ b/edkrepo_manifest_parser/edk_manifest.py @@ -44,10 +44,10 @@ NO_ASSOCIATED_REMOTE = 'There are no remotes associated with the ClientGitHook entry:\nsource:{} destination:{}' \ '\nThis hook will not be installed, updated or deleted.\n' NO_REMOTE_EXISTS_WITH_NAME = 'There are no remotes with the name: {} listed in the manifest file.' -PIN_COMB_ERROR = "Pin \"{}\" Pin did not have a single tag." +PIN_COMBO_ERROR = "Pin \"{}\" Pin did not have a single tag." DUPLICATE_TAG_ERROR = "Duplicate <{}> tag not allowed: '{}' (Note: check 's" -COMB_INVALIDINPUT_ERROR = "Invalid input: {} not found in 'combinations' property" -COMB_UNKOWN_ERROR = "Could not find a Combination named '{}' in '{}'" +COMBO_INVALIDINPUT_ERROR = "Invalid input: {} not found in 'combinations' property" +COMBO_UNKNOWN_ERROR = "Could not find a Combination named '{}' in '{}'" ATTRIBUTE_MISSING_ERROR = "Missing required attribute. Must specify either 'branch' or 'commit' for each ." GENERAL_CONFIG_MISSING_ERROR = "Unable to locate " SOURCELIST_EMPTY_ERROR = "Invalid input: empty values in source list" @@ -268,7 +268,7 @@ def __init__(self, fileref): else: combos = self._tree.findall('Combination') if len(combos) != 1: - raise KeyError(PIN_COMB_ERROR.format(fileref)) + raise KeyError(PIN_COMBO_ERROR.format(fileref)) # container tag not required for pin files if self._tree.find('CombinationList') is None: @@ -395,7 +395,7 @@ def get_repo_sources(self, combo_name): # default combo return self._tuple_list(self._combo_sources[self.general_config.default_combo]) else: - raise ValueError(COMB_INVALIDINPUT_ERROR.format(combo_name)) + raise ValueError(COMBO_INVALIDINPUT_ERROR.format(combo_name)) @property def repo_hooks(self): @@ -439,7 +439,7 @@ def get_combo_element(self, name): for combo in combinations.iter(tag='Combination'): if combo.attrib['name'] == name: return copy.deepcopy(combo) - raise ValueError(COMB_UNKOWN_ERROR.format(name, self._fileref)) + raise ValueError(COMBO_UNKNOWN_ERROR.format(name, self._fileref)) @property def commit_templates(self): From e88c46798ba494a61c7cd0f2316477c23e5872d5 Mon Sep 17 00:00:00 2001 From: Ashley E Desimone Date: Mon, 3 Oct 2022 10:33:17 -0700 Subject: [PATCH 02/23] EdkRepo: Add support for defining and manipulating patch sets in the manifest file Description Signed-off-by: Harsh Vora Co-authored-by: Ashley DeSimone Co-authored-by:Harsh Vora Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo_manifest_parser/edk_manifest.py | 146 +++++++++++++++++++++++- 1 file changed, 143 insertions(+), 3 deletions(-) diff --git a/edkrepo_manifest_parser/edk_manifest.py b/edkrepo_manifest_parser/edk_manifest.py index 726e7f9..d77bf23 100644 --- a/edkrepo_manifest_parser/edk_manifest.py +++ b/edkrepo_manifest_parser/edk_manifest.py @@ -27,8 +27,14 @@ RepoHook = namedtuple('RepoHook', ['source', 'dest_path', 'dest_file', 'remote_url']) Combination = namedtuple('Combination', ['name', 'description', 'venv_enable']) RepoSource = namedtuple('RepoSource', ['root', 'remote_name', 'remote_url', 'branch', 'commit', 'sparse', +<<<<<<< HEAD 'enable_submodule', 'tag', 'venv_cfg']) +======= + 'enable_submodule', 'tag', 'patch_set']) +PatchSet = namedtuple('PatchSet', ['remote', 'name', 'parent_sha', 'fetch_branch']) +PatchOperation = namedtuple('PatchOperation',['type', 'file', 'sha', 'source_remote', 'source_branch']) +>>>>>>> a3b5d3d1 (EdkRepo: Add support for defining and manipulating patch sets in the manifest file) SparseSettings = namedtuple('SparseSettings', ['sparse_by_default']) SparseData = namedtuple('SparseData', ['combination', 'remote_name', 'always_include', 'always_exclude']) @@ -49,12 +55,15 @@ COMBO_INVALIDINPUT_ERROR = "Invalid input: {} not found in 'combinations' property" COMBO_UNKNOWN_ERROR = "Could not find a Combination named '{}' in '{}'" ATTRIBUTE_MISSING_ERROR = "Missing required attribute. Must specify either 'branch' or 'commit' for each ." +INVALID_COMBO_DEFINITION_ERROR = "Can not specify branch or commit or tag along with patchSet" GENERAL_CONFIG_MISSING_ERROR = "Unable to locate " SOURCELIST_EMPTY_ERROR = "Invalid input: empty values in source list" INVALID_PROJECTNAME_ERROR = "Invalid input: {} not found in CiIndexXml" UNSUPPORTED_TYPE_ERROR = "{} is not a supported xml type: {}" INVALID_XML_ERROR = "{} is not a valid xml file ({})" - +PATCHSET_UNKNOWN_ERROR = "Could not find a PatchSet named '{}' in '{}'" +REMOTE_DIFFERENT_ERROR = "The remote for patchset {}/{} is different from {}/{}" +NO_PATCHSET_IN_COMBO = "The Combination: {} does not have any patchsets." class BaseXmlHelper(): def __init__(self, fileref, xml_types): @@ -197,6 +206,8 @@ def __init__(self, fileref): self._folder_to_folder_mappings = [] # List of FolderToFolderMapping objects self._submodule_alternate_remotes = [] self._submodule_init_list = [] + self._patch_sets = {} + self._patch_set_operations = {} # # Append include XML's to the Manifest etree before parsing @@ -328,6 +339,17 @@ def __init__(self, fileref): for f2f_mapping in subroot.iter(tag='FolderToFolderMapping'): self._folder_to_folder_mappings.append(_FolderToFolderMapping(f2f_mapping)) + # + # Process tag + # + subroot = self._tree.find('PatchSets') + if subroot is not None: + for patchset in subroot.iter(tag='PatchSet'): + self._patch_sets[patchset.attrib['name']] =_PatchSet(patchset).tuple + operations = [] + for subelem in patchset: + operations.append(_PatchSetOperations(subelem).tuple) + self._patch_set_operations[patchset.attrib['name']] = operations return def is_pin_file(self): @@ -751,6 +773,110 @@ def __eq__(self, other): def __ne__(self, other): return not self.__eq__(other) + @property + def get_all_patchsets(self): + ''' + Returns a list of all the patchsets defined in the manifest file + ''' + patchsets = [] + for patch in self._patch_sets.keys(): + patchsets.append(self._patch_sets[patch]) + return patchsets + + def get_patchset(self, name): + for patch in self._patch_sets.keys(): + if patch == name: + return self._patch_sets[patch] + + def get_patchsets_for_combo(self, combo=None): + patchsets = {} + if combo is None: + sources = self._combo_sources + for combo, source in sources.items(): + for reposource in source: + if reposource.patch_set is not None: + patchsets[combo]=self.get_patchset(reposource.patch_set) + return patchsets + else: + sources = self._combo_sources[combo] + for reposource in sources: + if reposource.patch_set is not None: + patchsets[combo]=self.get_patchset(reposource.patch_set) + + if len(patchsets): + return patchsets + else: + raise KeyError(NO_PATCHSET_IN_COMBO.format(combo)) + + + def get_parent_patchset_operations(self, name, patch_set_operations): + ''' + This method takes the input name and a list for storing the operations as its parameters. It recursively + calls itself to check if there is a parent patchset for the given patchset and if there is, it checks if + the remotes are same. If not, it throws a ValueError. These operations are appended to the patch_set_operations + list. + ''' + parent_sha = self._patch_sets[name][2] + if parent_sha in self._patch_sets: + if self._patch_sets[parent_sha][0] == self._patch_sets[name][0]: + self.get_parent_patchset_operations(parent_sha, patch_set_operations) + patch_set_operations.append(self._patch_set_operations[parent_sha]) + else: + raise ValueError(REMOTE_DIFFERENT_ERROR.format(parent_sha, self._patch_sets[parent_sha][0], + name, self._patch_sets[name][0])) + + def get_patchset_operations(self, name=None): + ''' + This method returns a list of patchset operations. If name of the patchset is provided as a parameter, + it gives the operations of that patchset otherwise operations of all patchsets are returned. + The parent patchset's operations are listed first if there are any. + ''' + if name: + patch_set_operations = [] + if name in self._patch_sets: + self.get_parent_patchset_operations(name, patch_set_operations) + patch_set_operations.append(self._patch_set_operations[name]) + return patch_set_operations + raise ValueError(PATCHSET_UNKNOWN_ERROR.format(name, self._fileref)) + else: + return self._patch_set_operations +class _PatchSet(): + def __init__(self, element): + try: + self.remote = element.attrib['remote'] + self.name = element.attrib['name'] + self.parentSha = element.attrib['parentSha'] + self.fetchBranch = element.attrib['fetchBranch'] + except KeyError as k: + raise KeyError(REQUIRED_ATTRIB_ERROR_MSG.format(k, element.tag)) + + @property + def tuple(self): + return PatchSet(self.remote, self.name, self.parentSha, self.fetchBranch) + +class _PatchSetOperations(): + def __init__(self, element): + self.type = element.tag + try: + self.file = element.attrib['file'] + except KeyError as k: + self.file = None + try: + self.sha = element.attrib['sha'] + except KeyError as k: + self.sha = None + try: + self.source_remote = element.attrib['sourceRemote'] + except KeyError as k: + self.source_remote = None + try: + self.source_branch = element.attrib['sourceBranch'] + except KeyError as k: + self.source_branch = None + + @property + def tuple(self): + return PatchOperation(self.type, self.file, self.sha, self.source_remote, self.source_branch) class _ProjectInfo(): def __init__(self, element): @@ -893,6 +1019,10 @@ def __init__(self, element, remotes): self.tag = element.attrib['tag'] except Exception: self.tag = None + try: + self.patch_set = element.attrib['patchSet'] + except Exception: + self.patch_set = None try: # if the sparse attrib is not explicitly set to true, then assume false self.sparse = (element.attrib['sparseCheckout'].lower() == 'true') @@ -912,13 +1042,16 @@ def __init__(self, element, remotes): except: self.venv_cfg = None - if self.branch is None and self.commit is None and self.tag is None: + if self.branch is None and self.commit is None and self.tag is None and self.patch_set is None: raise KeyError(ATTRIBUTE_MISSING_ERROR) + if self.patch_set is not None and (self.branch is not None or self.commit is not None or self.tag is not None): + raise ValueError(INVALID_COMBO_DEFINITION_ERROR) + @property def tuple(self): return RepoSource(self.root, self.remote_name, self.remote_url, self.branch, - self.commit, self.sparse, self.enableSub, self.tag, self.venv_cfg) + self.commit, self.sparse, self.enableSub, self.tag, self.venv_cfg, self.patch_set) class _SparseSettings(): @@ -1187,6 +1320,13 @@ def main(): test_manifest.write_current_combo('TESTCOMBO', 'TestManifest.xml') print('Updated current combo: {}'.format(test_manifest.general_config.current_combo)) + print('\nPatchsets') + print(test_manifest.get_all_patchsets) + print('\nPatchset Operations\n') + print(test_manifest.get_patchset_operations()) + print(test_manifest.get_all_patchsets_in_combos()) + print(test_manifest.get_patchset('test')) + print(separator_string) if test_manifest.is_pin_file(): print('Successfully parsed {} as a pin file.\nExiting...'.format(args.InputFile)) From 1076ce52e2a2024b198a34966afa625bf21314bd Mon Sep 17 00:00:00 2001 From: Ashley E Desimone Date: Mon, 3 Oct 2022 11:44:16 -0700 Subject: [PATCH 03/23] EdkRepo: Add a method to get the path to a manifest repo Define get_manifest_repo_path() to get the absolute path to a given manifest repository. Update the clone, checkout, and sync command to use this method instead of manually computing the path. Signed-off-by: Ashley E Desimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/commands/checkout_command.py | 4 ++++ edkrepo/commands/clone_command.py | 4 +++- edkrepo/commands/sync_command.py | 7 ++++--- edkrepo/common/edkrepo_exception.py | 3 +++ .../humble/manifest_repos_maintenance_humble.py | 3 ++- .../manifest_repos_maintenance.py | 15 ++++++++++++++- 6 files changed, 30 insertions(+), 6 deletions(-) diff --git a/edkrepo/commands/checkout_command.py b/edkrepo/commands/checkout_command.py index 2ce26c0..6369f33 100644 --- a/edkrepo/commands/checkout_command.py +++ b/edkrepo/commands/checkout_command.py @@ -20,6 +20,7 @@ from edkrepo.common.common_repo_functions import checkout, combination_is_in_manifest from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException from edkrepo.config.config_factory import get_workspace_manifest +from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import get_manifest_repo_path class CheckoutCommand(EdkrepoCommand): @@ -44,5 +45,8 @@ def get_metadata(self): def run_command(self, args, config): if combination_is_in_manifest(args.Combination, get_workspace_manifest()): checkout(args.Combination, args.verbose, args.override, get_repo_cache_obj(config)) + manifest = get_workspace_manifest() + manifest_repo = manifest.general_config.source_manifest_repo + global_manifest_path = get_manifest_repo_path(manifest_repo, config) else: raise EdkrepoInvalidParametersException(humble.NO_COMBO.format(args.Combination)) diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py index 34d4821..d16b4e5 100644 --- a/edkrepo/commands/clone_command.py +++ b/edkrepo/commands/clone_command.py @@ -27,7 +27,7 @@ from edkrepo.common.workspace_maintenance.workspace_maintenance import case_insensitive_single_match from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import pull_all_manifest_repos, find_project_in_all_indices from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos -from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo +from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo, get_manifest_repo_path from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import PROJ_NOT_IN_REPO, SOURCE_MANIFEST_REPO_NOT_FOUND import edkrepo.common.ui_functions as ui_functions from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml @@ -106,6 +106,8 @@ def run_command(self, args, config): except EdkrepoManifestNotFoundException: raise EdkrepoInvalidParametersException(CLONE_INVALID_PROJECT_ARG) + manifest_repository_path = get_manifest_repo_path(manifest_repo, config) + # If this manifest is in a defined manifest repository validate the manifest within the manifest repo if manifest_repo in cfg: verify_single_manifest(config['cfg_file'], manifest_repo, global_manifest_path) diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py index 09154b2..38076ac 100644 --- a/edkrepo/commands/sync_command.py +++ b/edkrepo/commands/sync_command.py @@ -50,7 +50,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import pull_workspace_manifest_repo from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import pull_all_manifest_repos from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo -from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos +from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos, get_manifest_repo_path from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest, get_edkrepo_global_data_directory from edkrepo.config.config_factory import get_workspace_manifest_file from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME @@ -100,12 +100,13 @@ def run_command(self, args, config): except: pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'], False) source_global_manifest_repo = find_source_manifest_repo(initial_manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo) + + global_manifest_directory = get_manifest_repo_path(source_global_manifest_repo, config) + cfg_manifest_repos, user_cfg_manifest_repos, conflicts = list_available_manifest_repos(config['cfg_file'], config['user_cfg_file']) if source_global_manifest_repo in cfg_manifest_repos: - global_manifest_directory = config['cfg_file'].manifest_repo_abs_path(source_global_manifest_repo) verify_single_manifest(config['cfg_file'], source_global_manifest_repo, get_workspace_manifest_file(), args.verbose) elif source_global_manifest_repo in user_cfg_manifest_repos: - global_manifest_directory = config['user_cfg_file'].manifest_repo_abs_path(source_global_manifest_repo) verify_single_manifest(config['user_cfg_file'], source_global_manifest_repo, get_workspace_manifest_file(), args.verbose) else: global_manifest_directory = None diff --git a/edkrepo/common/edkrepo_exception.py b/edkrepo/common/edkrepo_exception.py index e29b011..c291baa 100644 --- a/edkrepo/common/edkrepo_exception.py +++ b/edkrepo/common/edkrepo_exception.py @@ -110,3 +110,6 @@ class EdkrepoInvalidConfigOptionException(EdkrepoException): def __init__(self, message): super().__init__(message, 125) +class EdkrepoManifestRepoNotFoundException(EdkrepoException): + def __init__(self, message): + super().__initi__(message, 126) diff --git a/edkrepo/common/workspace_maintenance/humble/manifest_repos_maintenance_humble.py b/edkrepo/common/workspace_maintenance/humble/manifest_repos_maintenance_humble.py index 290b463..00126e6 100644 --- a/edkrepo/common/workspace_maintenance/humble/manifest_repos_maintenance_humble.py +++ b/edkrepo/common/workspace_maintenance/humble/manifest_repos_maintenance_humble.py @@ -26,4 +26,5 @@ 'This global manifest repository will not be downloaded or updated. ' 'Resolve the conflict and then re-run the failed operation') SOURCE_MANIFEST_REPO_NOT_FOUND = 'Could not determine the source global manifest repository for project: {}' -PROJ_NOT_IN_REPO = 'Project: {} does not exist in any global manifest repository' \ No newline at end of file +PROJ_NOT_IN_REPO = 'Project: {} does not exist in any global manifest repository' +MANIFEST_REPO_NOT_FOUND = 'The manifest repository: {} was not found' \ No newline at end of file diff --git a/edkrepo/common/workspace_maintenance/manifest_repos_maintenance.py b/edkrepo/common/workspace_maintenance/manifest_repos_maintenance.py index 5ae1f7d..fbd6bea 100644 --- a/edkrepo/common/workspace_maintenance/manifest_repos_maintenance.py +++ b/edkrepo/common/workspace_maintenance/manifest_repos_maintenance.py @@ -17,7 +17,7 @@ import edkrepo.config.config_factory as cfg from edkrepo.config.tool_config import CI_INDEX_FILE_NAME from edkrepo.common.edkrepo_exception import EdkrepoUncommitedChangesException, EdkrepoInvalidParametersException -from edkrepo.common.edkrepo_exception import EdkrepoManifestNotFoundException +from edkrepo.common.edkrepo_exception import EdkrepoManifestNotFoundException, EdkrepoManifestRepoNotFoundException from edkrepo.common.progress_handler import GitProgressHandler import edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble as humble from edkrepo.common.workspace_maintenance.workspace_maintenance import generate_name_for_obsolete_backup @@ -293,3 +293,16 @@ def pull_workspace_manifest_repo(project_manifest, edkrepo_cfg, edkrepo_user_cfg reset_hard) elif src_man_repo in conflicts: raise EdkrepoInvalidParametersException(humble.CONFLICT_NO_CLONE.format(src_man_repo)) + +def get_manifest_repo_path(manifest_repo, config): + '''Calculates the absolute path for the provided manifest repo. Raises an + EdkRepoManifestRepoNotFound exception otherwise. + ''' + + cfg_manifest_repos, user_cfg_manifest_repos, conflicts = list_available_manifest_repos(config['cfg_file'], config['user_cfg_file']) + if manifest_repo in cfg_manifest_repos: + return config['cfg_file'].manifest_repo_abs_path(manifest_repo) + elif manifest_repo in user_cfg_manifest_repos: + return config['user_cfg_file'].manifest_repo_abs_path(manifest_repo) + else: + raise EdkrepoManifestRepoNotFoundException(humble.MANIFEST_REPO_NOT_FOUND.format(manifest_repo)) From 3b95858d9cd6c9716779dd66b8b23d05b753893e Mon Sep 17 00:00:00 2001 From: Ashley E Desimone Date: Mon, 3 Oct 2022 13:25:46 -0700 Subject: [PATCH 04/23] EdkRepo: Update combo command to show patch set In the case where a patch set is listed in a combo the combo command should show the name of the patch set instead of "None" Signed-off-by: Ashley E Desimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/commands/combo_command.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/edkrepo/commands/combo_command.py b/edkrepo/commands/combo_command.py index 538bd7e..990c07f 100644 --- a/edkrepo/commands/combo_command.py +++ b/edkrepo/commands/combo_command.py @@ -52,4 +52,7 @@ def run_command(self, args, config): sources = manifest.get_repo_sources(combo) length = len(max([source.root for source in sources], key=len)) for source in sources: - ui_functions.print_info_msg(" {} : {}".format(source.root.ljust(length), source.branch), header=False) + if source.branch: + ui_functions.print_info_msg(" {} : {}".format(source.root.ljust(length), source.branch), header=False) + elif source.patch_set: + ui_functions.print_info_msg(" {} : {}".format(source.root.ljust(length), source.patch_set), header=False) From 9e589a6640c0d1d63ac0e9d89390ccc8f6b05e4c Mon Sep 17 00:00:00 2001 From: Ashley E Desimone Date: Mon, 3 Oct 2022 13:31:01 -0700 Subject: [PATCH 05/23] EdkRepo: Make get_unique_branch_name() common Move the implementation of get_unique_branch_name() to commont_repo_functions.py from f2f_cherry_pick_command.py and add an import of it in f2f_cherry_pick_command.py. Signed-off-by: Ashley E Desimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/commands/f2f_cherry_pick_command.py | 11 +---------- edkrepo/common/common_repo_functions.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/edkrepo/commands/f2f_cherry_pick_command.py b/edkrepo/commands/f2f_cherry_pick_command.py index 8184305..74abbf2 100644 --- a/edkrepo/commands/f2f_cherry_pick_command.py +++ b/edkrepo/commands/f2f_cherry_pick_command.py @@ -20,7 +20,7 @@ from git import Repo from colorama import Fore -from edkrepo.common.common_repo_functions import sparse_checkout_enabled, get_full_path +from edkrepo.common.common_repo_functions import sparse_checkout_enabled, get_full_path, get_unique_branch_name from edkrepo.commands.edkrepo_command import EdkrepoCommand from edkrepo.common.edkrepo_exception import EdkrepoAbortCherryPickException, EdkrepoInvalidParametersException, EdkrepoWorkspaceInvalidException from edkrepo.common.edkrepo_exception import EdkrepoNotFoundException, EdkrepoGitException @@ -423,15 +423,6 @@ def get_common_folder_name(folder1, folder2, config): else: return '' -def get_unique_branch_name(branch_name_prefix, repo): - branch_names = [x.name for x in repo.heads] - if branch_name_prefix not in branch_names: - return branch_name_prefix - index = 1 - while True: - branch_name = "{}-{}".format(branch_name_prefix, index) - if branch_name not in branch_names: - return branch_name def cherry_pick_operations_to_include_folder_list(cherry_pick_operations): include_folder_list = [] diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index c7d27e0..6d9005b 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -671,3 +671,14 @@ def find_git_version(): cur_git_version = GitVersion(cur_git_ver_string) return cur_git_version + +def get_unique_branch_name(branch_name_prefix, repo): + branch_names = [x.name for x in repo.heads] + if branch_name_prefix not in branch_names: + return branch_name_prefix + index = 1 + while True: + branch_name = "{}-{}".format(branch_name_prefix, index) + if branch_name not in branch_names: + return branch_name + From f13177853224a81ccb0344ed112ec12afa2e7dbe Mon Sep 17 00:00:00 2001 From: Ashley E Desimone Date: Mon, 3 Oct 2022 13:41:29 -0700 Subject: [PATCH 06/23] EdkRepo: Add method to get the hash of a file Add implementation of get_hash_of_file() to common_repo_functions.py. Signed-off-by: Harsh Vora Co-authored-by: Ashley DeSimone Co-authored-by:Harsh Vora Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/common/common_repo_functions.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index 6d9005b..8be01a2 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -15,6 +15,7 @@ import urllib.request import subprocess import traceback +import hashlib import git from git import Repo @@ -682,3 +683,14 @@ def get_unique_branch_name(branch_name_prefix, repo): if branch_name not in branch_names: return branch_name + +def get_hash_of_file(file): + sha256 = hashlib.sha256() + with open(file, 'rb') as f: + while True: + chunk = f.read(65536) + if not chunk: + break + sha256.update(chunk) + + return sha256.hexdigest() From 9bd5cde508cb5fd0d5aa23b2dc81eedea777c896 Mon Sep 17 00:00:00 2001 From: Ashley E Desimone Date: Tue, 4 Oct 2022 11:03:40 -0700 Subject: [PATCH 07/23] EdkRepo: Add support methods to generate local branches based on patchsets Signed-off-by: Harsh Vora Signed-off-by: Ashley DeSimone Co-authored-by: Ashley DeSimone Co-authored-by:Harsh Vora Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/common/common_repo_functions.py | 135 +++++++++++++++++++++++- edkrepo/common/edkrepo_exception.py | 40 +++++++ edkrepo/common/humble.py | 11 ++ edkrepo_manifest_parser/edk_manifest.py | 7 +- 4 files changed, 186 insertions(+), 7 deletions(-) diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index 8be01a2..e7361a4 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -8,6 +8,7 @@ # import collections +import json import os import shutil import sys @@ -19,14 +20,21 @@ import git from git import Repo +from datetime import date import colorama -from edkrepo.config.config_factory import get_edkrepo_global_data_directory +from edkrepo.common.edkrepo_exception import EdkrepoRevertFailedException, EdkrepoCherryPickFailedException +from edkrepo.common.edkrepo_exception import EdkrepoFetchBranchNotFoundException, EdkrepoBranchExistsException +from edkrepo.common.edkrepo_exception import EdkrepoPatchNotFoundException, EdkrepoPatchFailedException +from edkrepo.common.edkrepo_exception import EdkrepoRemoteNotFoundException, EdkrepoRemoteAddException, EdkrepoRemoteRemoveException from edkrepo.common.edkrepo_exception import EdkrepoManifestInvalidException from edkrepo.common.edkrepo_exception import EdkrepoUncommitedChangesException from edkrepo.common.edkrepo_exception import EdkrepoVerificationException from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException from edkrepo.common.progress_handler import GitProgressHandler +from edkrepo.common.humble import APPLYING_CHERRY_PICK_FAILED, APPLYING_PATCH_FAILED, APPLYING_REVERT_FAILED +from edkrepo.common.humble import BRANCH_EXISTS, FETCH_BRANCH_DOES_NOT_EXIST, PATCHFILE_DOES_NOT_EXIST +from edkrepo.common.humble import REMOTE_CREATION_FAILED, REMOTE_NOT_FOUND, REMOVE_REMOTE_FAILED from edkrepo.common.humble import MISSING_BRANCH_COMMIT from edkrepo.common.humble import UNCOMMITED_CHANGES, CHECKOUT_UNCOMMITED_CHANGES from edkrepo.common.humble import CHECKING_OUT_COMMIT, CHECKING_CONNECTION @@ -340,6 +348,29 @@ def check_branches(sources, workspace_path): except: raise EdkrepoManifestInvalidException(CHECKOUT_NO_REMOTE.format(repo_to_check.root)) +def check_branch_name_collision(json_path, patch_set, repo): + if patch_set in repo.branches: + repo_name = repo.working_tree_dir + repo_name = repo_name.split("\\")[-1] + collision = False + for branch in repo.branches: + if str(branch) == patch_set: + with open(json_path, 'r') as f: + data = json.load(f) + patchset_data = data[repo_name] + for patchset in patchset_data: + if patch_set in patchset.values(): + repo.git.checkout(patch_set) + if patchset['head_sha'] != repo.git.execute(['git', 'rev-parse', 'HEAD']): + branch.rename(patch_set + '_' + date.today().strftime("%Y/%m/%d")) + patchset['head_sha'] = repo.git.execute(['git', 'rev-parse', 'HEAD']) + patchset[patch_set] = patch_set + '_' + date.today().strftime("%Y/%m/%d") + f.seek(0) + json.dump(data, f, indent=4) + collision = True + break + f.close() + return collision def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifest): if not override: @@ -694,3 +725,105 @@ def get_hash_of_file(file): sha256.update(chunk) return sha256.hexdigest() + +def create_local_branch(name, patchset, global_manifest_path, manifest_obj, repo): + for branch in repo.branches: + if name == str(branch): + raise EdkrepoBranchExistsException(BRANCH_EXISTS.format(name)) + + path = repo.working_tree_dir + repo_path = os.path.dirname(path) + path = os.path.join(repo_path, "repo") + json_path = os.path.join(path, "patchset_{}.json".format(os.path.basename(repo.working_dir))) + remote_list = manifest_obj.remotes + operations_list = manifest_obj.get_patchset_operations(patchset.name) + REMOTE_IN_REMOTE_LIST = False + for remote in remote_list: + if patchset.remote == remote.name: + REMOTE_IN_REMOTE_LIST = True + try: + repo.remotes.origin.fetch(patchset.fetch_branch, progress=GitProgressHandler()) + except: + raise EdkrepoFetchBranchNotFoundException(FETCH_BRANCH_DOES_NOT_EXIST.format(patchset.fetch_branch)) + name = name.replace(' ', '_') + repo.git.checkout(patchset.parent_sha, b=name) + try: + apply_patchset_operations(repo, remote, operations_list, global_manifest_path, remote_list) + head_sha = repo.git.execute(['git', 'rev-parse', 'HEAD']) + except (EdkrepoPatchFailedException, EdkrepoRevertFailedException, git.GitCommandError, EdkrepoCherryPickFailedException) as exception: + print(exception) + json_str = { + patchset.name: name, + "head_sha": head_sha + } + for operations in operations_list: + for operation in operations: + if operation.type == "Patch": + json_str[operation.file] = get_hash_of_file(os.path.normpath(os.path.join(global_manifest_path, operation.file))) + if not os.path.isfile(json_path): + with open(json_path, 'w') as f: + json.dump({os.path.basename(repo.working_dir): [json_str]}, f, indent=4) + f.close() + else: + with open(json_path, "r+") as f: + data = json.load(f) + data[os.path.basename(repo.working_dir)].append(json_str) + f.seek(0) + json.dump(data, f, indent=4) + f.close() + + # repo.git.execute(['git', 'notes', '--ref', 'refs/patch_sets', 'append', '-m', json.dumps(json_str, indent=4)]) + if not REMOTE_IN_REMOTE_LIST: + raise EdkrepoRemoteNotFoundException(REMOTE_NOT_FOUND.format(patchset.remote)) + +def apply_patchset_operations(repo, remote, operations_list, global_manifest_path, remote_list): + for operations in operations_list: + for operation in operations: + if operation.type == "Patch": + path = os.path.normpath(os.path.join(global_manifest_path, operation.file)) + if os.path.isfile(path): + try: + repo.git.execute(['git', 'am', path, '--ignore-whitespace']) + except: + raise EdkrepoPatchFailedException(APPLYING_PATCH_FAILED.format(operation.file)) + else: + raise EdkrepoPatchNotFoundException(PATCHFILE_DOES_NOT_EXIST.format(operation.file)) + elif operation.type == "Revert": + try: + repo.git.execute(['git', 'revert', operation.sha, '--no-edit']) + except: + raise EdkrepoRevertFailedException(APPLYING_REVERT_FAILED.format(operation.sha)) + else: + if operation.source_remote: + REMOTE_FOUND = False + for remote in remote_list: + if operation.source_remote == remote.name: + REMOTE_FOUND = True + try: + repo.git.execute(['git', 'remote', 'add', operation.source_remote, remote.url]) + except : + raise EdkrepoRemoteAddException(REMOTE_CREATION_FAILED.format(operation.source_remote)) + try: + repo.git.execute(['git', 'fetch', operation.source_remote, operation.source_branch]) + except: + raise EdkrepoFetchBranchNotFoundException(FETCH_BRANCH_DOES_NOT_EXIST.format(operation.source_branch)) + try: + repo.git.cherry_pick(operation.sha) + except: + raise EdkrepoCherryPickFailedException(APPLYING_CHERRY_PICK_FAILED.format(operation.sha)) + try: + repo.git.execute(['git', 'remote', 'remove', operation.source_remote]) + except: + raise EdkrepoRemoteRemoveException(REMOVE_REMOTE_FAILED.format(operation.source_remote)) + if not REMOTE_FOUND: + raise EdkrepoRemoteNotFoundException(REMOTE_NOT_FOUND.format(operation.source_remote)) + else: + try: + repo.remotes.origin.fetch(operation.source_branch) + except: + raise EdkrepoFetchBranchNotFoundException(FETCH_BRANCH_DOES_NOT_EXIST.format(operation.source_branch)) + try: + repo.git.cherry_pick(operation.sha) + except: + raise EdkrepoCherryPickFailedException(APPLYING_CHERRY_PICK_FAILED.format(operation.sha)) + diff --git a/edkrepo/common/edkrepo_exception.py b/edkrepo/common/edkrepo_exception.py index c291baa..9acd262 100644 --- a/edkrepo/common/edkrepo_exception.py +++ b/edkrepo/common/edkrepo_exception.py @@ -113,3 +113,43 @@ def __init__(self, message): class EdkrepoManifestRepoNotFoundException(EdkrepoException): def __init__(self, message): super().__initi__(message, 126) +class EdkrepoRevertFailedException(EdkrepoException): + def __init__(self, message): + super().__init__(message, 127) + +class EdkrepoCherryPickFailedException(EdkrepoException): + def __init__(self, message): + super().__init__(message, 128) + +class EdkrepoBranchExistsException(EdkrepoException): + def __init__(self, message): + super().__init__(message, 129) + +class EdkrepoFetchBranchNotFoundException(EdkrepoException): + def __init__(self, message): + super().__init__(message, 130) + +class EdkrepoRemoteNotFoundException(EdkrepoException): + def __init__(self, message): + super().__init__(message, 131) + +class EdkrepoPatchNotFoundException(EdkrepoException): + def __init__(self, message): + super().__init__(message, 132) + +class EdkrepoRemoteAddException(EdkrepoException): + def __init__(self, message): + super().__init__(message, 133) + +class EdkrepoRemoteRemoveException(EdkrepoException): + def __init__(self, message): + super().__init__(message, 134) + +class EdkrepoManifestRepoNotFoundException(EdkrepoException): + def __init__(self, message): + super().__init__(message, 135) + +class EdkrepoPatchFailedException(EdkrepoException): + def __init__(self, message): + super().__init__(message, 136) + diff --git a/edkrepo/common/humble.py b/edkrepo/common/humble.py index 29d5511..9772b7d 100644 --- a/edkrepo/common/humble.py +++ b/edkrepo/common/humble.py @@ -163,3 +163,14 @@ # Common submodule error messages SUBMODULE_DEINIT_FAILED = 'Warning: Unable to remove all submodule content' + +# Creating Local Branch Error Messages +BRANCH_EXISTS = "The branch: {} already exists." +REMOTE_NOT_FOUND = "Could not find the remote: {}" +REMOTE_CREATION_FAILED = "Failed to add the remote: {}" +FETCH_BRANCH_DOES_NOT_EXIST = "The branch {} does not exist" +PATCHFILE_DOES_NOT_EXIST = "The patch file {} does not exist" +APPLYING_PATCH_FAILED = "Unable to apply the patch file {}" +APPLYING_REVERT_FAILED = "Failed to revert to the commit: {}" +APPLYING_CHERRY_PICK_FAILED = "Failed to cherry pick the commit: {}" +REMOVE_REMOTE_FAILED = "Failed to remove the remote: {}" \ No newline at end of file diff --git a/edkrepo_manifest_parser/edk_manifest.py b/edkrepo_manifest_parser/edk_manifest.py index d77bf23..a40bb9d 100644 --- a/edkrepo_manifest_parser/edk_manifest.py +++ b/edkrepo_manifest_parser/edk_manifest.py @@ -27,14 +27,9 @@ RepoHook = namedtuple('RepoHook', ['source', 'dest_path', 'dest_file', 'remote_url']) Combination = namedtuple('Combination', ['name', 'description', 'venv_enable']) RepoSource = namedtuple('RepoSource', ['root', 'remote_name', 'remote_url', 'branch', 'commit', 'sparse', -<<<<<<< HEAD - 'enable_submodule', 'tag', 'venv_cfg']) - -======= - 'enable_submodule', 'tag', 'patch_set']) + 'enable_submodule', 'tag', 'venv_cfg', 'patch_set']) PatchSet = namedtuple('PatchSet', ['remote', 'name', 'parent_sha', 'fetch_branch']) PatchOperation = namedtuple('PatchOperation',['type', 'file', 'sha', 'source_remote', 'source_branch']) ->>>>>>> a3b5d3d1 (EdkRepo: Add support for defining and manipulating patch sets in the manifest file) SparseSettings = namedtuple('SparseSettings', ['sparse_by_default']) SparseData = namedtuple('SparseData', ['combination', 'remote_name', 'always_include', 'always_exclude']) From 504fade9f0d0b3e3d43a6e551b0b43d4df625db6 Mon Sep 17 00:00:00 2001 From: Ashley E Desimone Date: Tue, 4 Oct 2022 11:21:39 -0700 Subject: [PATCH 08/23] EdkRepo: Update cloning functions to handle patch set branches Signed-off-by: Harsh Vora Co-authored-by: Ashley DeSimone Co-authored-by:Harsh Vora Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/commands/clone_command.py | 3 ++- edkrepo/common/common_repo_functions.py | 33 +++++++++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py index d16b4e5..5b94a8f 100644 --- a/edkrepo/commands/clone_command.py +++ b/edkrepo/commands/clone_command.py @@ -10,6 +10,7 @@ import os import shutil import sys +from webbrowser import get from edkrepo.commands.edkrepo_command import EdkrepoCommand from edkrepo.commands.edkrepo_command import SubmoduleSkipArgument, SourceManifestRepoArgument @@ -178,7 +179,7 @@ def run_command(self, args, config): cache_obj = get_repo_cache_obj(config) if cache_obj is not None: add_missing_cache_repos(cache_obj, manifest, args.verbose) - clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest, cache_obj) + clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest, cache_obj, manifest_repository_path) # Init submodules if not args.skip_submodule: diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index e7361a4..a4cb7bc 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -88,7 +88,8 @@ PRIMARY_REMOTE_NAME = 'primary' -def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest, cache_obj=None): +def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest, global_manifest_path, cache_obj=None): + created_patch_sets = [] for repo_to_clone in repos_to_clone: local_repo_path = os.path.join(workspace_dir, repo_to_clone.root) local_repo_url = repo_to_clone.remote_url @@ -109,10 +110,14 @@ def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, # Fetch notes repo.remotes.origin.fetch("refs/notes/*:refs/notes/*") - # Handle branch/commit/tag checkout if needed. If a combination of these are specified the - # order of importance is 1)commit 2)tag 3)branch with only the higest priority being checked - # out - if repo_to_clone.commit: + # Handle patchset/branch/commit/tag checkout if needed. While checking out, patchset has the highest priority. + # If patchset is not present then, if a combination of these are specified the + # order of importance is 1)commit 2)tag 3)branch with only the higest priority being checked out + if repo_to_clone.patch_set: + patchset = manifest.get_patchset(repo_to_clone.patch_set) + created_patch_sets.append(repo_to_clone.patch_set) + create_local_branch(repo_to_clone.patch_set, patchset, global_manifest_path, manifest, repo) + elif repo_to_clone.commit: if args.verbose and (repo_to_clone.branch or repo_to_clone.tag): ui_functions.print_info_msg(MULTIPLE_SOURCE_ATTRIBUTES_SPECIFIED.format(repo_to_clone.root)) repo.git.checkout(repo_to_clone.commit) @@ -155,6 +160,24 @@ def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, update_repo_commit_template(workspace_dir, repo, repo_to_clone, config, global_manifest_directory) + # Create patch set branches + patchsets_in_manifest = manifest.get_patchsets_for_combo() + if patchsets_in_manifest: + default_combo_repo = None + + for combo, patch in patchsets_in_manifest.items(): + for repo_to_clone in manifest.get_repo_sources(combo): + if getattr(repo_to_clone, "patch_set"): + repo = Repo(os.path.join(workspace_dir, repo_to_clone.root)) + if getattr(patch, "name") not in created_patch_sets: + patch = manifest.get_patchset(getattr(repo_to_clone, "patch_set")) + create_local_branch(getattr(repo_to_clone, "patch_set"), patch, global_manifest_path, manifest, repo) + else: + default_combo_repo = repo + + if default_combo_repo: + default_combo_repo.git.checkout(created_patch_sets[0]) + def write_included_config(remotes, submodule_alt_remotes, repo_directory): included_configs = [] for remote in remotes: From 6ee962db9459733b1abdbdaa30f9d4dcd2ed6728 Mon Sep 17 00:00:00 2001 From: Ashley E Desimone Date: Tue, 4 Oct 2022 11:23:09 -0700 Subject: [PATCH 09/23] EdkRepo: Update checkout functionality to accomodate patch set branches Signed-off-by: Harsh Vora Co-authored-by: Ashley DeSimone Co-authored-by:Harsh Vora Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/commands/checkout_command.py | 4 +- edkrepo/common/common_repo_functions.py | 77 ++++++++++++++----------- 2 files changed, 46 insertions(+), 35 deletions(-) diff --git a/edkrepo/commands/checkout_command.py b/edkrepo/commands/checkout_command.py index 6369f33..38eafd7 100644 --- a/edkrepo/commands/checkout_command.py +++ b/edkrepo/commands/checkout_command.py @@ -43,10 +43,10 @@ def get_metadata(self): return metadata def run_command(self, args, config): - if combination_is_in_manifest(args.Combination, get_workspace_manifest()): - checkout(args.Combination, args.verbose, args.override, get_repo_cache_obj(config)) manifest = get_workspace_manifest() manifest_repo = manifest.general_config.source_manifest_repo global_manifest_path = get_manifest_repo_path(manifest_repo, config) + if combination_is_in_manifest(args.Combination, manifest): + checkout(args.Combination, global_manifest_path, args.verbose, args.override, get_repo_cache_obj(config)) else: raise EdkrepoInvalidParametersException(humble.NO_COMBO.format(args.Combination)) diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index a4cb7bc..ce53ca9 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -395,13 +395,14 @@ def check_branch_name_collision(json_path, patch_set, repo): f.close() return collision -def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifest): + +def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifest, global_manifest_path): if not override: try: check_dirty_repos(manifest, workspace_path) except EdkrepoUncommitedChangesException: raise EdkrepoUncommitedChangesException(CHECKOUT_UNCOMMITED_CHANGES) - check_branches(repos_to_checkout, workspace_path) + #check_branches(repos_to_checkout, workspace_path) for repo_to_checkout in repos_to_checkout: if verbose: if repo_to_checkout.branch is not None and repo_to_checkout.commit is None: @@ -410,38 +411,48 @@ def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifes print(CHECKING_OUT_COMMIT.format(repo_to_checkout.commit, repo_to_checkout.root)) local_repo_path = os.path.join(workspace_path, repo_to_checkout.root) repo = Repo(local_repo_path) + json_path = os.path.join(workspace_path, "repo") + json_path = os.path.join(json_path, "patchset_{}.json".format(repo_to_checkout.root)) + if repo_to_checkout.patch_set: + collision = check_branch_name_collision(json_path, repo_to_checkout.patch_set, repo) + patchset = manifest.get_patchset(repo_to_checkout.patch_set) + if collision: + create_local_branch(repo_to_checkout.patch_set, patchset, global_manifest_path, manifest, repo) + elif not collision and repo_to_checkout.patch_set not in repo.branches: + create_local_branch(repo_to_checkout.patch_set, patchset, global_manifest_path, manifest, repo) # Checkout the repo onto the correct branch/commit/tag if multiple attributes are provided in # the source section for the manifest the order of priority is the followiwng 1)commit # 2) tag 3)branch with the highest priority attribute provided beinng checked out - if repo_to_checkout.commit: - if verbose and (repo_to_checkout.branch or repo_to_checkout.tag): - print(MULTIPLE_SOURCE_ATTRIBUTES_SPECIFIED.format(repo_to_checkout.root)) - if override: - repo.git.checkout(repo_to_checkout.commit, '--force') - else: - repo.git.checkout(repo_to_checkout.commit) - elif repo_to_checkout.tag and repo_to_checkout.commit is None: - if verbose and (repo_to_checkout.branch): - print(TAG_AND_BRANCH_SPECIFIED.format(repo_to_checkout.root)) - if override: - repo.git.checkout(repo_to_checkout.tag, '--force') - else: - repo.git.checkout(repo_to_checkout.tag) - elif repo_to_checkout.branch and (repo_to_checkout.commit is None and repo_to_checkout.tag is None): - branch_name = repo_to_checkout.branch - if branch_name in repo.heads: - local_branch = repo.heads[branch_name] - else: - local_branch = repo.create_head(branch_name, repo.remotes['origin'].refs[branch_name]) - #check to see if the branch being checked out has a tracking branch if not set one up - if repo.heads[local_branch.name].tracking_branch() is None: - repo.heads[local_branch.name].set_tracking_branch(repo.remotes['origin'].refs[branch_name]) - if override: - repo.heads[local_branch.name].checkout(force=True) - else: - repo.heads[local_branch.name].checkout() else: - raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT) + if repo_to_checkout.commit: + if verbose and (repo_to_checkout.branch or repo_to_checkout.tag): + print(MULTIPLE_SOURCE_ATTRIBUTES_SPECIFIED.format(repo_to_checkout.root)) + if override: + repo.git.checkout(repo_to_checkout.commit, '--force') + else: + repo.git.checkout(repo_to_checkout.commit) + elif repo_to_checkout.tag and repo_to_checkout.commit is None: + if verbose and (repo_to_checkout.branch): + print(TAG_AND_BRANCH_SPECIFIED.format(repo_to_checkout.root)) + if override: + repo.git.checkout(repo_to_checkout.tag, '--force') + else: + repo.git.checkout(repo_to_checkout.tag) + elif repo_to_checkout.branch and (repo_to_checkout.commit is None and repo_to_checkout.tag is None): + branch_name = repo_to_checkout.branch + if branch_name in repo.heads: + local_branch = repo.heads[branch_name] + else: + local_branch = repo.create_head(branch_name, repo.remotes['origin'].refs[branch_name]) + #check to see if the branch being checked out has a tracking branch if not set one up + if repo.heads[local_branch.name].tracking_branch() is None: + repo.heads[local_branch.name].set_tracking_branch(repo.remotes['origin'].refs[branch_name]) + if override: + repo.heads[local_branch.name].checkout(force=True) + else: + repo.heads[local_branch.name].checkout() + else: + raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT) def validate_manifest_repo(manifest_repo, verbose=False, archived=False): print(VERIFY_GLOBAL) @@ -499,7 +510,7 @@ def combination_is_in_manifest(combination, manifest): return combination in combination_names -def checkout(combination, verbose=False, override=False, log=None, cache_obj=None): +def checkout(combination, global_manifest_path, verbose=False, override=False, log=None, cache_obj=None): workspace_path = get_workspace_path() manifest = get_workspace_manifest() @@ -558,7 +569,7 @@ def checkout(combination, verbose=False, override=False, log=None, cache_obj=Non print(CHECKING_OUT_COMBO.format(combo)) try: - checkout_repos(verbose, override, repo_sources, workspace_path, manifest) + checkout_repos(verbose, override, repo_sources, workspace_path, manifest, global_manifest_path) current_repos = repo_sources # Update the current checkout combo in the manifest only if this # combination exists in the manifest @@ -569,7 +580,7 @@ def checkout(combination, verbose=False, override=False, log=None, cache_obj=Non traceback.print_exc() print (CHECKOUT_COMBO_UNSUCCESSFULL.format(combo)) # Return to the initial combo, since there was an issue with cheking out the selected combo - checkout_repos(verbose, override, initial_repo_sources, workspace_path, manifest) + checkout_repos(verbose, override, initial_repo_sources, workspace_path, manifest, global_manifest_path) finally: cache_path = None if cache_obj is not None: From 1a419f75f3552dc933353e2a4384522627c087f2 Mon Sep 17 00:00:00 2001 From: Ashley E Desimone Date: Tue, 4 Oct 2022 11:24:01 -0700 Subject: [PATCH 10/23] EdkRepo: Clean up imports in common_repo_functions.py Signed-off-by: Harsh Vora Co-authored-by: Ashley DeSimone Co-authored-by:Harsh Vora Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/common/common_repo_functions.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index ce53ca9..f00f91e 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -39,27 +39,20 @@ from edkrepo.common.humble import UNCOMMITED_CHANGES, CHECKOUT_UNCOMMITED_CHANGES from edkrepo.common.humble import CHECKING_OUT_COMMIT, CHECKING_CONNECTION from edkrepo.common.humble import CHECKING_OUT_BRANCH -from edkrepo.common.humble import VERIFY_ERROR_HEADER -from edkrepo.common.humble import VERIFY_EXCEPTION_MSG -from edkrepo.common.humble import INDEX_DUPLICATE_NAMES -from edkrepo.common.humble import LOAD_MANIFEST_FAILED -from edkrepo.common.humble import MANIFEST_NAME_INCONSISTENT from edkrepo.common.humble import CHECKOUT_NO_REMOTE from edkrepo.common.humble import SPARSE_CHECKOUT from edkrepo.common.humble import SPARSE_RESET from edkrepo.common.humble import CHECKING_OUT_COMBO from edkrepo.common.humble import CHECKOUT_INVALID_COMBO from edkrepo.common.humble import CHECKOUT_COMBO_UNSUCCESSFULL -from edkrepo.common.humble import GEN_A_NOT_IN_B, GEN_FOUND_MULT_A_IN_B from edkrepo.common.humble import COMMIT_TEMPLATE_NOT_FOUND, COMMIT_TEMPLATE_CUSTOM_VALUE from edkrepo.common.humble import COMMIT_TEMPLATE_RESETTING_VALUE -from edkrepo.common.humble import ADD_PRIMARY_REMOTE, REMOVE_PRIMARY_REMOTE -from edkrepo.common.humble import FETCH_PRIMARY_REMOTE, MIRROR_PRIMARY_SHA, TAG_AND_BRANCH_SPECIFIED +from edkrepo.common.humble import TAG_AND_BRANCH_SPECIFIED from edkrepo.common.humble import MIRROR_BEHIND_PRIMARY_REPO, HOOK_NOT_FOUND_ERROR, SUBMODULE_FAILURE from edkrepo.common.humble import INCLUDED_URL_LINE, INCLUDED_INSTEAD_OF_LINE, INCLUDED_FILE_NAME from edkrepo.common.humble import ERROR_WRITING_INCLUDE, MULTIPLE_SOURCE_ATTRIBUTES_SPECIFIED from edkrepo.common.humble import VERIFY_GLOBAL, VERIFY_ARCHIVED, VERIFY_PROJ, VERIFY_PROJ_FAIL -from edkrepo.common.humble import VERIFY_PROJ_NOT_IN_INDEX, VERIFY_GLOBAL_FAIL +from edkrepo.common.humble import VERIFY_GLOBAL_FAIL from edkrepo.common.humble import SUBMODULE_DEINIT_FAILED from edkrepo.common.pathfix import get_actual_path, expanduser from edkrepo.common.git_version import GitVersion @@ -70,8 +63,7 @@ from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml -from edkrepo.common.edkrepo_exception import EdkrepoNotFoundException, EdkrepoGitException, EdkrepoWarningException -from edkrepo.common.edkrepo_exception import EdkrepoFoundMultipleException, EdkrepoHookNotFoundException +from edkrepo.common.edkrepo_exception import EdkrepoHookNotFoundException from edkrepo.common.edkrepo_exception import EdkrepoGitConfigSetupException, EdkrepoManifestInvalidException, EdkrepoManifestNotFoundException from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo, list_available_manifest_repos from edkrepo.common.workspace_maintenance.workspace_maintenance import case_insensitive_single_match From 653af06100d8339f2d5816339b08cd9bf598aae6 Mon Sep 17 00:00:00 2001 From: Vora Date: Fri, 7 Oct 2022 11:36:01 -0700 Subject: [PATCH 11/23] Edkrepo: Enabled using same names for patchsets by adding remote along with the name in keys - Added remotes along with name of the patchset which enables having multiple patch files having same names as they are now connected with remotes for uniqueness. Signed-off-by: Harsh Vora harsh.vora@intel.com Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo_manifest_parser/edk_manifest.py | 62 +++++++++++++------------ 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/edkrepo_manifest_parser/edk_manifest.py b/edkrepo_manifest_parser/edk_manifest.py index a40bb9d..78cad75 100644 --- a/edkrepo_manifest_parser/edk_manifest.py +++ b/edkrepo_manifest_parser/edk_manifest.py @@ -59,6 +59,7 @@ PATCHSET_UNKNOWN_ERROR = "Could not find a PatchSet named '{}' in '{}'" REMOTE_DIFFERENT_ERROR = "The remote for patchset {}/{} is different from {}/{}" NO_PATCHSET_IN_COMBO = "The Combination: {} does not have any patchsets." +NO_PATCHSET_EXISTS = "The Patchset: {} does not exist" class BaseXmlHelper(): def __init__(self, fileref, xml_types): @@ -340,11 +341,11 @@ def __init__(self, fileref): subroot = self._tree.find('PatchSets') if subroot is not None: for patchset in subroot.iter(tag='PatchSet'): - self._patch_sets[patchset.attrib['name']] =_PatchSet(patchset).tuple + self._patch_sets[(patchset.attrib['name'], getattr(_PatchSet(patchset).tuple, "remote"))] =_PatchSet(patchset).tuple operations = [] for subelem in patchset: operations.append(_PatchSetOperations(subelem).tuple) - self._patch_set_operations[patchset.attrib['name']] = operations + self._patch_set_operations[(patchset.attrib['name'], getattr(_PatchSet(patchset).tuple, "remote"))] = operations return def is_pin_file(self): @@ -778,10 +779,13 @@ def get_all_patchsets(self): patchsets.append(self._patch_sets[patch]) return patchsets - def get_patchset(self, name): - for patch in self._patch_sets.keys(): - if patch == name: - return self._patch_sets[patch] + def get_patchset(self, name, remote): + if (name, remote) in self._patch_sets.keys(): + for patchset in self._patch_sets.keys(): + if patchset[0] == name and patchset[1] == remote: + return self._patch_sets[(name, remote)] + else: + raise KeyError(NO_PATCHSET_EXISTS.format(name)) def get_patchsets_for_combo(self, combo=None): patchsets = {} @@ -790,13 +794,13 @@ def get_patchsets_for_combo(self, combo=None): for combo, source in sources.items(): for reposource in source: if reposource.patch_set is not None: - patchsets[combo]=self.get_patchset(reposource.patch_set) + patchsets[combo]=self.get_patchset(reposource.patch_set, reposource.remote_name) return patchsets else: sources = self._combo_sources[combo] for reposource in sources: if reposource.patch_set is not None: - patchsets[combo]=self.get_patchset(reposource.patch_set) + patchsets[combo]=self.get_patchset(reposource.patch_set, reposource.remote_name) if len(patchsets): return patchsets @@ -804,37 +808,35 @@ def get_patchsets_for_combo(self, combo=None): raise KeyError(NO_PATCHSET_IN_COMBO.format(combo)) - def get_parent_patchset_operations(self, name, patch_set_operations): + def get_parent_patchset_operations(self, name, remote, patch_set_operations): ''' This method takes the input name and a list for storing the operations as its parameters. It recursively calls itself to check if there is a parent patchset for the given patchset and if there is, it checks if the remotes are same. If not, it throws a ValueError. These operations are appended to the patch_set_operations list. ''' - parent_sha = self._patch_sets[name][2] - if parent_sha in self._patch_sets: - if self._patch_sets[parent_sha][0] == self._patch_sets[name][0]: - self.get_parent_patchset_operations(parent_sha, patch_set_operations) - patch_set_operations.append(self._patch_set_operations[parent_sha]) + parent_sha = self._patch_sets[(name, remote)][2] + if (parent_sha, remote) in self._patch_sets: + if self._patch_sets[(parent_sha, remote)][0] == self._patch_sets[(name, remote)][0]: + self.get_parent_patchset_operations(parent_sha, remote, patch_set_operations) + patch_set_operations.append(self._patch_set_operations[(parent_sha, remote)]) else: raise ValueError(REMOTE_DIFFERENT_ERROR.format(parent_sha, self._patch_sets[parent_sha][0], name, self._patch_sets[name][0])) - def get_patchset_operations(self, name=None): + def get_patchset_operations(self, name, remote): ''' This method returns a list of patchset operations. If name of the patchset is provided as a parameter, it gives the operations of that patchset otherwise operations of all patchsets are returned. The parent patchset's operations are listed first if there are any. ''' - if name: - patch_set_operations = [] - if name in self._patch_sets: - self.get_parent_patchset_operations(name, patch_set_operations) - patch_set_operations.append(self._patch_set_operations[name]) - return patch_set_operations - raise ValueError(PATCHSET_UNKNOWN_ERROR.format(name, self._fileref)) - else: - return self._patch_set_operations + patch_set_operations = [] + if (name, remote) in self._patch_sets: + self.get_parent_patchset_operations(name, remote, patch_set_operations) + patch_set_operations.append(self._patch_set_operations[(name, remote)]) + return patch_set_operations + raise ValueError(PATCHSET_UNKNOWN_ERROR.format(name, self._fileref)) + class _PatchSet(): def __init__(self, element): try: @@ -1315,12 +1317,14 @@ def main(): test_manifest.write_current_combo('TESTCOMBO', 'TestManifest.xml') print('Updated current combo: {}'.format(test_manifest.general_config.current_combo)) - print('\nPatchsets') + print('\nPatchsets:') print(test_manifest.get_all_patchsets) - print('\nPatchset Operations\n') - print(test_manifest.get_patchset_operations()) - print(test_manifest.get_all_patchsets_in_combos()) - print(test_manifest.get_patchset('test')) + print('\nTest Patchset:') + print(test_manifest.get_patchset("test", "firmware.boot.uefi.iafw.devops.ci.cr.tools")) + print('\nPatchset Operations:\n') + print(test_manifest.get_patchset_operations("test", "firmware.boot.uefi.iafw.devops.ci.cr.tools")) + print('\nPatchsets for a specific combo:') + print(test_manifest.get_patchsets_for_combo("edkrepo_3.0.3")) print(separator_string) if test_manifest.is_pin_file(): From 0fbe230109ec28ea6d6713f1bf6581f3480fcfa3 Mon Sep 17 00:00:00 2001 From: Vora Date: Fri, 7 Oct 2022 11:50:44 -0700 Subject: [PATCH 12/23] Edkrepo: Removing unused imports and variables from clone file Signed-off-by: Harsh Vora harsh.vora@intel.com Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/commands/clone_command.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py index 5b94a8f..5c0c20f 100644 --- a/edkrepo/commands/clone_command.py +++ b/edkrepo/commands/clone_command.py @@ -10,7 +10,6 @@ import os import shutil import sys -from webbrowser import get from edkrepo.commands.edkrepo_command import EdkrepoCommand from edkrepo.commands.edkrepo_command import SubmoduleSkipArgument, SourceManifestRepoArgument @@ -31,7 +30,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo, get_manifest_repo_path from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import PROJ_NOT_IN_REPO, SOURCE_MANIFEST_REPO_NOT_FOUND import edkrepo.common.ui_functions as ui_functions -from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml +from edkrepo_manifest_parser.edk_manifest import ManifestXml from project_utils.submodule import maintain_submodules from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME @@ -77,7 +76,6 @@ def get_metadata(self): def run_command(self, args, config): pull_all_manifest_repos(config['cfg_file'], config['user_cfg_file'], False) - name_or_manifest = args.ProjectNameOrManifestFile workspace_dir = args.Workspace # Check to see if requested workspace exists. If not create it. If so check for empty if workspace_dir == '.': From 75f61ec612aa6a21987297026084cf8de55e5576 Mon Sep 17 00:00:00 2001 From: Vora Date: Fri, 7 Oct 2022 12:56:56 -0700 Subject: [PATCH 13/23] Edkrepo: Removing unused imports from checkout Signed-off-by: Harsh Vora harsh.vora@intel.com Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/commands/checkout_command.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/edkrepo/commands/checkout_command.py b/edkrepo/commands/checkout_command.py index 38eafd7..2f0805f 100644 --- a/edkrepo/commands/checkout_command.py +++ b/edkrepo/commands/checkout_command.py @@ -6,11 +6,6 @@ # Copyright (c) 2017- 2020, Intel Corporation. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent # -# Standard modules -import sys -import os - -# Third-Party modules # Our modules from edkrepo.commands.edkrepo_command import EdkrepoCommand, OverrideArgument From 860f22b9eb051c7f31289d5e365c13193c6de3e4 Mon Sep 17 00:00:00 2001 From: Vora Date: Fri, 7 Oct 2022 13:04:59 -0700 Subject: [PATCH 14/23] Edkrepo: Removed unused imports and unused parameters from common_repo_functions - Made changes to sync accordingly Signed-off-by: Harsh Vora harsh.vora@intel.com Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/commands/sync_command.py | 2 +- edkrepo/common/common_repo_functions.py | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py index 38076ac..3c577d6 100644 --- a/edkrepo/commands/sync_command.py +++ b/edkrepo/commands/sync_command.py @@ -231,7 +231,7 @@ def run_command(self, args, config): # Update commit message templates if global_manifest_directory is not None: - update_repo_commit_template(workspace_path, repo, repo_to_sync, config, global_manifest_directory) + update_repo_commit_template(workspace_path, repo, repo_to_sync, global_manifest_directory) if sync_error: ui_functions.print_error_msg(SYNC_ERROR, header = False) diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index f00f91e..6be443e 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -7,12 +7,10 @@ # SPDX-License-Identifier: BSD-2-Clause-Patent # -import collections import json import os import shutil import sys -import unicodedata import urllib.request import subprocess import traceback @@ -29,7 +27,6 @@ from edkrepo.common.edkrepo_exception import EdkrepoRemoteNotFoundException, EdkrepoRemoteAddException, EdkrepoRemoteRemoveException from edkrepo.common.edkrepo_exception import EdkrepoManifestInvalidException from edkrepo.common.edkrepo_exception import EdkrepoUncommitedChangesException -from edkrepo.common.edkrepo_exception import EdkrepoVerificationException from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException from edkrepo.common.progress_handler import GitProgressHandler from edkrepo.common.humble import APPLYING_CHERRY_PICK_FAILED, APPLYING_PATCH_FAILED, APPLYING_REVERT_FAILED @@ -48,7 +45,7 @@ from edkrepo.common.humble import COMMIT_TEMPLATE_NOT_FOUND, COMMIT_TEMPLATE_CUSTOM_VALUE from edkrepo.common.humble import COMMIT_TEMPLATE_RESETTING_VALUE from edkrepo.common.humble import TAG_AND_BRANCH_SPECIFIED -from edkrepo.common.humble import MIRROR_BEHIND_PRIMARY_REPO, HOOK_NOT_FOUND_ERROR, SUBMODULE_FAILURE +from edkrepo.common.humble import MIRROR_BEHIND_PRIMARY_REPO, HOOK_NOT_FOUND_ERROR from edkrepo.common.humble import INCLUDED_URL_LINE, INCLUDED_INSTEAD_OF_LINE, INCLUDED_FILE_NAME from edkrepo.common.humble import ERROR_WRITING_INCLUDE, MULTIPLE_SOURCE_ATTRIBUTES_SPECIFIED from edkrepo.common.humble import VERIFY_GLOBAL, VERIFY_ARCHIVED, VERIFY_PROJ, VERIFY_PROJ_FAIL @@ -62,7 +59,7 @@ from edkrepo.config.tool_config import CI_INDEX_FILE_NAME from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException -from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml +from edkrepo_manifest_parser.edk_manifest import ManifestXml from edkrepo.common.edkrepo_exception import EdkrepoHookNotFoundException from edkrepo.common.edkrepo_exception import EdkrepoGitConfigSetupException, EdkrepoManifestInvalidException, EdkrepoManifestNotFoundException from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo, list_available_manifest_repos @@ -149,7 +146,7 @@ def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, install_hooks(project_client_side_hooks, local_repo_path, repo_to_clone, config, global_manifest_directory) # Add the commit template if it exists. - update_repo_commit_template(workspace_dir, repo, repo_to_clone, config, global_manifest_directory) + update_repo_commit_template(workspace_dir, repo, repo_to_clone, global_manifest_directory) # Create patch set branches @@ -602,7 +599,7 @@ def get_full_path(file_name): return file_path return None -def update_repo_commit_template(workspace_dir, repo, repo_info, config, global_manifest_directory): +def update_repo_commit_template(workspace_dir, repo, repo_info, global_manifest_directory): # Open the local manifest and get any templates manifest = edk_manifest.ManifestXml(os.path.join(workspace_dir, 'repo', 'Manifest.xml')) templates = manifest.commit_templates From f863e78f603f08004a95fcb43d3aaddd06469614 Mon Sep 17 00:00:00 2001 From: Vora Date: Fri, 7 Oct 2022 13:10:29 -0700 Subject: [PATCH 15/23] Edkrepo: Removing unused imports and variables from Sync file Signed-off-by: Harsh Vora harsh.vora@intel.com Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/commands/sync_command.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py index 3c577d6..c54a094 100644 --- a/edkrepo/commands/sync_command.py +++ b/edkrepo/commands/sync_command.py @@ -20,18 +20,17 @@ # Our modules from edkrepo.commands.edkrepo_command import EdkrepoCommand -from edkrepo.commands.edkrepo_command import DryRunArgument, SubmoduleSkipArgument, SourceManifestRepoArgument +from edkrepo.commands.edkrepo_command import SubmoduleSkipArgument, SourceManifestRepoArgument import edkrepo.commands.arguments.sync_args as arguments -from edkrepo.common.progress_handler import GitProgressHandler -from edkrepo.common.edkrepo_exception import EdkrepoUncommitedChangesException, EdkrepoManifestNotFoundException +from edkrepo.common.edkrepo_exception import EdkrepoManifestNotFoundException from edkrepo.common.edkrepo_exception import EdkrepoManifestChangedException -from edkrepo.common.humble import SYNC_UNCOMMITED_CHANGES, SYNC_MANIFEST_NOT_FOUND, SYNC_URL_CHANGE, SYNC_COMBO_CHANGE +from edkrepo.common.humble import SYNC_MANIFEST_NOT_FOUND from edkrepo.common.humble import SYNC_SOURCE_MOVE_WARNING, SYNC_REMOVE_WARNING, SYNC_REMOVE_LIST_END_FORMATTING from edkrepo.common.humble import SYNC_MANIFEST_DIFF_WARNING, SYNC_MANIFEST_UPDATE from edkrepo.common.humble import SPARSE_RESET, SPARSE_CHECKOUT, SYNC_REPO_CHANGE, SYNCING, FETCHING, UPDATING_MANIFEST from edkrepo.common.humble import NO_SYNC_DETACHED_HEAD, SYNC_COMMITS_ON_TARGET, SYNC_ERROR from edkrepo.common.humble import MIRROR_BEHIND_PRIMARY_REPO, SYNC_NEEDS_REBASE, INCLUDED_FILE_NAME -from edkrepo.common.humble import SYNC_BRANCH_CHANGE_ON_LOCAL, SYNC_INCOMPATIBLE_COMBO +from edkrepo.common.humble import SYNC_BRANCH_CHANGE_ON_LOCAL from edkrepo.common.humble import SYNC_REBASE_CALC_FAIL, SYNC_MOVE_FAILED from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import SOURCE_MANIFEST_REPO_NOT_FOUND from edkrepo.common.pathfix import get_actual_path, expanduser @@ -51,7 +50,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import pull_all_manifest_repos from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos, get_manifest_repo_path -from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest, get_edkrepo_global_data_directory +from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest from edkrepo.config.config_factory import get_workspace_manifest_file from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml @@ -293,7 +292,6 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path # Does the current combo exist in the new manifest? If not check to see if you can use the repo sources from # the default combo - initial_combos = combinations_in_manifest(initial_manifest) new_combos = combinations_in_manifest(new_manifest_to_check) if current_combo not in new_combos: new_sources_for_current_combo = new_manifest_to_check.get_repo_sources(new_manifest_to_check.general_config.default_combo) From 363dec157220fd542ecae094f5778ea18228cfb9 Mon Sep 17 00:00:00 2001 From: Harsh Vora Date: Fri, 7 Oct 2022 14:08:19 -0700 Subject: [PATCH 16/23] Edkrepo: Bug fixes and improvements while creating local branch via clone - Added the remote_name along with name for getting patchsets - Removed an extra occurence of get_unique_branch_name() - Check out to the branch if the branch already exists instead of raising an exception and hence deleted the error messages from import and humble - Enabled checking out correctly when name of patchset is present insted of parent sha - Used return statements at appropriate places so the branch creation flow is not executed if already checked out to the branch or in case of exception in applying patchset operations. - Checking out the default branch if patchset operations fail and deleting the newly created branch Signed-off-by: Harsh Vora harsh.vora@intel.com Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/common/common_repo_functions.py | 40 +++++++++++++++---------- edkrepo/common/humble.py | 4 +-- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index 6be443e..4496675 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -22,15 +22,15 @@ import colorama from edkrepo.common.edkrepo_exception import EdkrepoRevertFailedException, EdkrepoCherryPickFailedException -from edkrepo.common.edkrepo_exception import EdkrepoFetchBranchNotFoundException, EdkrepoBranchExistsException +from edkrepo.common.edkrepo_exception import EdkrepoFetchBranchNotFoundException from edkrepo.common.edkrepo_exception import EdkrepoPatchNotFoundException, EdkrepoPatchFailedException from edkrepo.common.edkrepo_exception import EdkrepoRemoteNotFoundException, EdkrepoRemoteAddException, EdkrepoRemoteRemoveException from edkrepo.common.edkrepo_exception import EdkrepoManifestInvalidException from edkrepo.common.edkrepo_exception import EdkrepoUncommitedChangesException from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException from edkrepo.common.progress_handler import GitProgressHandler -from edkrepo.common.humble import APPLYING_CHERRY_PICK_FAILED, APPLYING_PATCH_FAILED, APPLYING_REVERT_FAILED -from edkrepo.common.humble import BRANCH_EXISTS, FETCH_BRANCH_DOES_NOT_EXIST, PATCHFILE_DOES_NOT_EXIST +from edkrepo.common.humble import APPLYING_CHERRY_PICK_FAILED, APPLYING_PATCH_FAILED, APPLYING_REVERT_FAILED, CHECKING_OUT_DEFAULT +from edkrepo.common.humble import FETCH_BRANCH_DOES_NOT_EXIST, PATCHFILE_DOES_NOT_EXIST from edkrepo.common.humble import REMOTE_CREATION_FAILED, REMOTE_NOT_FOUND, REMOVE_REMOTE_FAILED from edkrepo.common.humble import MISSING_BRANCH_COMMIT from edkrepo.common.humble import UNCOMMITED_CHANGES, CHECKOUT_UNCOMMITED_CHANGES @@ -103,8 +103,8 @@ def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, # If patchset is not present then, if a combination of these are specified the # order of importance is 1)commit 2)tag 3)branch with only the higest priority being checked out if repo_to_clone.patch_set: - patchset = manifest.get_patchset(repo_to_clone.patch_set) - created_patch_sets.append(repo_to_clone.patch_set) + patchset = manifest.get_patchset(repo_to_clone.patch_set, repo_to_clone.remote_name) + created_patch_sets.append((repo_to_clone.patch_set, repo_to_clone.remote_name)) create_local_branch(repo_to_clone.patch_set, patchset, global_manifest_path, manifest, repo) elif repo_to_clone.commit: if args.verbose and (repo_to_clone.branch or repo_to_clone.tag): @@ -158,14 +158,14 @@ def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, for repo_to_clone in manifest.get_repo_sources(combo): if getattr(repo_to_clone, "patch_set"): repo = Repo(os.path.join(workspace_dir, repo_to_clone.root)) - if getattr(patch, "name") not in created_patch_sets: - patch = manifest.get_patchset(getattr(repo_to_clone, "patch_set")) + if (getattr(patch, "name"), getattr(repo_to_clone, "remote_name")) not in created_patch_sets: + patch = manifest.get_patchset(getattr(repo_to_clone, "patch_set"), getattr(repo_to_clone, "remote_name")) create_local_branch(getattr(repo_to_clone, "patch_set"), patch, global_manifest_path, manifest, repo) else: default_combo_repo = repo - if default_combo_repo: - default_combo_repo.git.checkout(created_patch_sets[0]) + if default_combo_repo is not None: + default_combo_repo.git.checkout(created_patch_sets[0][0]) def write_included_config(remotes, submodule_alt_remotes, repo_directory): included_configs = [] @@ -768,21 +768,34 @@ def create_local_branch(name, patchset, global_manifest_path, manifest_obj, repo repo.remotes.origin.fetch(patchset.fetch_branch, progress=GitProgressHandler()) except: raise EdkrepoFetchBranchNotFoundException(FETCH_BRANCH_DOES_NOT_EXIST.format(patchset.fetch_branch)) - name = name.replace(' ', '_') - repo.git.checkout(patchset.parent_sha, b=name) + try: + parent_patchset = manifest_obj.get_patchset(patchset.parent_sha, patchset.remote) + repo.git.checkout(parent_patchset[2], b=name) + except: + repo.git.checkout(patchset.parent_sha, b=name) try: apply_patchset_operations(repo, remote, operations_list, global_manifest_path, remote_list) head_sha = repo.git.execute(['git', 'rev-parse', 'HEAD']) except (EdkrepoPatchFailedException, EdkrepoRevertFailedException, git.GitCommandError, EdkrepoCherryPickFailedException) as exception: print(exception) + print(CHECKING_OUT_DEFAULT) + repo.git.checkout(os.path.basename(repo.git.execute(['git', 'symbolic-ref', 'refs/remotes/origin/HEAD']))) + repo.git.execute(['git', 'branch', '-D', '{}'.format(name)]) + return + + if not REMOTE_IN_REMOTE_LIST: + raise EdkrepoRemoteNotFoundException(REMOTE_NOT_FOUND.format(patchset.remote)) + json_str = { patchset.name: name, "head_sha": head_sha } + for operations in operations_list: for operation in operations: if operation.type == "Patch": json_str[operation.file] = get_hash_of_file(os.path.normpath(os.path.join(global_manifest_path, operation.file))) + if not os.path.isfile(json_path): with open(json_path, 'w') as f: json.dump({os.path.basename(repo.working_dir): [json_str]}, f, indent=4) @@ -795,10 +808,6 @@ def create_local_branch(name, patchset, global_manifest_path, manifest_obj, repo json.dump(data, f, indent=4) f.close() - # repo.git.execute(['git', 'notes', '--ref', 'refs/patch_sets', 'append', '-m', json.dumps(json_str, indent=4)]) - if not REMOTE_IN_REMOTE_LIST: - raise EdkrepoRemoteNotFoundException(REMOTE_NOT_FOUND.format(patchset.remote)) - def apply_patchset_operations(repo, remote, operations_list, global_manifest_path, remote_list): for operations in operations_list: for operation in operations: @@ -849,4 +858,3 @@ def apply_patchset_operations(repo, remote, operations_list, global_manifest_pat repo.git.cherry_pick(operation.sha) except: raise EdkrepoCherryPickFailedException(APPLYING_CHERRY_PICK_FAILED.format(operation.sha)) - diff --git a/edkrepo/common/humble.py b/edkrepo/common/humble.py index 9772b7d..93addea 100644 --- a/edkrepo/common/humble.py +++ b/edkrepo/common/humble.py @@ -165,7 +165,6 @@ SUBMODULE_DEINIT_FAILED = 'Warning: Unable to remove all submodule content' # Creating Local Branch Error Messages -BRANCH_EXISTS = "The branch: {} already exists." REMOTE_NOT_FOUND = "Could not find the remote: {}" REMOTE_CREATION_FAILED = "Failed to add the remote: {}" FETCH_BRANCH_DOES_NOT_EXIST = "The branch {} does not exist" @@ -173,4 +172,5 @@ APPLYING_PATCH_FAILED = "Unable to apply the patch file {}" APPLYING_REVERT_FAILED = "Failed to revert to the commit: {}" APPLYING_CHERRY_PICK_FAILED = "Failed to cherry pick the commit: {}" -REMOVE_REMOTE_FAILED = "Failed to remove the remote: {}" \ No newline at end of file +REMOVE_REMOTE_FAILED = "Failed to remove the remote: {}" +CHECKING_OUT_DEFAULT = "Failed to apply one of the patchset operations. Checking out back to the default branch" \ No newline at end of file From e90b1e756e38c3e95fdd4f034439e8a10543a6d6 Mon Sep 17 00:00:00 2001 From: Harsh Vora Date: Fri, 7 Oct 2022 14:58:08 -0700 Subject: [PATCH 17/23] Edkrepo: Bug fixed and improvements while checking out combo - Resolved branch name collision while checking out - Improved the overall flow and solved some bugs Signed-off-by: Harsh Vora harsh.vora@intel.com Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/common/common_repo_functions.py | 90 ++++++++++++++----------- edkrepo/common/humble.py | 1 + 2 files changed, 52 insertions(+), 39 deletions(-) diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index 4496675..93d26ce 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -15,10 +15,10 @@ import subprocess import traceback import hashlib +import time import git from git import Repo -from datetime import date import colorama from edkrepo.common.edkrepo_exception import EdkrepoRevertFailedException, EdkrepoCherryPickFailedException @@ -29,7 +29,7 @@ from edkrepo.common.edkrepo_exception import EdkrepoUncommitedChangesException from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException from edkrepo.common.progress_handler import GitProgressHandler -from edkrepo.common.humble import APPLYING_CHERRY_PICK_FAILED, APPLYING_PATCH_FAILED, APPLYING_REVERT_FAILED, CHECKING_OUT_DEFAULT +from edkrepo.common.humble import APPLYING_CHERRY_PICK_FAILED, APPLYING_PATCH_FAILED, APPLYING_REVERT_FAILED, CHECKING_OUT_DEFAULT, CHECKING_OUT_PATCHSET from edkrepo.common.humble import FETCH_BRANCH_DOES_NOT_EXIST, PATCHFILE_DOES_NOT_EXIST from edkrepo.common.humble import REMOTE_CREATION_FAILED, REMOTE_NOT_FOUND, REMOVE_REMOTE_FAILED from edkrepo.common.humble import MISSING_BRANCH_COMMIT @@ -360,31 +360,6 @@ def check_branches(sources, workspace_path): except: raise EdkrepoManifestInvalidException(CHECKOUT_NO_REMOTE.format(repo_to_check.root)) -def check_branch_name_collision(json_path, patch_set, repo): - if patch_set in repo.branches: - repo_name = repo.working_tree_dir - repo_name = repo_name.split("\\")[-1] - collision = False - for branch in repo.branches: - if str(branch) == patch_set: - with open(json_path, 'r') as f: - data = json.load(f) - patchset_data = data[repo_name] - for patchset in patchset_data: - if patch_set in patchset.values(): - repo.git.checkout(patch_set) - if patchset['head_sha'] != repo.git.execute(['git', 'rev-parse', 'HEAD']): - branch.rename(patch_set + '_' + date.today().strftime("%Y/%m/%d")) - patchset['head_sha'] = repo.git.execute(['git', 'rev-parse', 'HEAD']) - patchset[patch_set] = patch_set + '_' + date.today().strftime("%Y/%m/%d") - f.seek(0) - json.dump(data, f, indent=4) - collision = True - break - f.close() - return collision - - def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifest, global_manifest_path): if not override: try: @@ -394,24 +369,20 @@ def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifes #check_branches(repos_to_checkout, workspace_path) for repo_to_checkout in repos_to_checkout: if verbose: - if repo_to_checkout.branch is not None and repo_to_checkout.commit is None: + if repo_to_checkout.patch_set: + print(CHECKING_OUT_PATCHSET.format(repo_to_checkout.patch_set, repo_to_checkout.root)) + elif repo_to_checkout.branch is not None and repo_to_checkout.commit is None: print(CHECKING_OUT_BRANCH.format(repo_to_checkout.branch, repo_to_checkout.root)) elif repo_to_checkout.commit is not None: print(CHECKING_OUT_COMMIT.format(repo_to_checkout.commit, repo_to_checkout.root)) local_repo_path = os.path.join(workspace_path, repo_to_checkout.root) repo = Repo(local_repo_path) - json_path = os.path.join(workspace_path, "repo") - json_path = os.path.join(json_path, "patchset_{}.json".format(repo_to_checkout.root)) + + # Checkout the repo onto the correct patchset/branch/commit/tag if multiple attributes are provided in + # the source section for the manifest the order of priority is the followiwng 1)patchset 2)commit + # 3) tag 4)branch with the highest priority attribute provided beinng checked out if repo_to_checkout.patch_set: - collision = check_branch_name_collision(json_path, repo_to_checkout.patch_set, repo) - patchset = manifest.get_patchset(repo_to_checkout.patch_set) - if collision: - create_local_branch(repo_to_checkout.patch_set, patchset, global_manifest_path, manifest, repo) - elif not collision and repo_to_checkout.patch_set not in repo.branches: - create_local_branch(repo_to_checkout.patch_set, patchset, global_manifest_path, manifest, repo) - # Checkout the repo onto the correct branch/commit/tag if multiple attributes are provided in - # the source section for the manifest the order of priority is the followiwng 1)commit - # 2) tag 3)branch with the highest priority attribute provided beinng checked out + patchset_application_flow(repo_to_checkout, repo, workspace_path, manifest, global_manifest_path) else: if repo_to_checkout.commit: if verbose and (repo_to_checkout.branch or repo_to_checkout.tag): @@ -443,6 +414,47 @@ def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifes else: raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT) +def patchset_application_flow(repo, repo_obj, workspace_path, manifest, global_manifest_path): + json_path = os.path.join(workspace_path, "repo") + json_path = os.path.join(json_path, "patchset_{}.json".format(os.path.basename(repo_obj.working_dir))) + patchset = manifest.get_patchset(repo.patch_set, repo.remote_name) + + if repo.patch_set in repo_obj.branches: + if is_collision(json_path, repo.patch_set, repo_obj, global_manifest_path): + create_local_branch(repo.patch_set, patchset, global_manifest_path, manifest, repo_obj) + else: + create_local_branch(repo.patch_set, patchset, global_manifest_path, manifest, repo_obj) + +def is_collision(json_path, patch_set, repo, global_manifest_path): + repo_name = repo.working_tree_dir + repo_name = repo_name.split("\\")[-1] + COLLISION = False + for branch in repo.branches: + if str(branch) == patch_set: + with open(json_path, 'r+') as f: + data = json.load(f) + patchset_data = data[repo_name] + for patchset in patchset_data: + if patch_set in patchset.values(): + repo.git.checkout(patch_set) + head = repo.git.execute(['git', 'rev-parse', 'HEAD']) + if patchset['head_sha'] != head: + patchset['head_sha'] = head + COLLISION = True + if len(patchset.keys()) == 3: + patch_file = list(patchset.keys())[-1] + hash_of_patch_file = get_hash_of_file(os.path.normpath(os.path.join(global_manifest_path, patch_file))) + if patchset[patch_file] != hash_of_patch_file: + patchset[patch_file] = hash_of_patch_file + COLLISION = True + if COLLISION: + branch.rename(patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S")) + patchset[patch_set] = patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S") + f.seek(0) + json.dump(data, f, indent=4) + return True + return False + def validate_manifest_repo(manifest_repo, verbose=False, archived=False): print(VERIFY_GLOBAL) if archived: diff --git a/edkrepo/common/humble.py b/edkrepo/common/humble.py index 93addea..6d076b9 100644 --- a/edkrepo/common/humble.py +++ b/edkrepo/common/humble.py @@ -87,6 +87,7 @@ CHECKING_OUT_COMBO = 'Checking out combination: {0} ...' CHECKING_OUT_BRANCH = 'Checking out {0} branch for {1} repo ...' CHECKING_OUT_COMMIT = 'Checking detached HEAD on commit {0} for {1} repo ...' +CHECKING_OUT_PATCHSET = 'Checking out {0} patchset for {1} repo ...' # Messages for config_factory.py MIRROR_PRIMARY_REPOS_MISSING = 'The edkrepo global configuration file missing [primary-repos] section.' From bf639450e3b781e163ac5da4f930d9c2cda76522 Mon Sep 17 00:00:00 2001 From: Harsh Vora Date: Fri, 7 Oct 2022 16:15:25 -0700 Subject: [PATCH 18/23] Edkrepo: Integrated Sync Command with Creating Local Branch flow - Implemented pathset similarity and patchset operation similarity checking functions in common_repo_functions - Made checks for the case where sources are same but patchset operations have changed and created new branches for the new patchset (using create_repos method implemented in common_repo_functions) while preserving the old branch - Did exception handling for updating the manifest because if there's error while updating, the local copy of manifest should not change - Bug fix of json not updated through create_repos - Small bug fix in the is_collision method of common_repo_functions - Bug Fix for global_manifest_path being different in sync file - Name change of _check_sha_tag_branch to _check_patchset_sha_tag_branch Signed-off-by: Harsh Vora harsh.vora@intel.com Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/commands/sync_command.py | 62 +++++++++++++++++++------ edkrepo/common/common_repo_functions.py | 42 ++++++++++++++++- edkrepo/common/humble.py | 3 +- 3 files changed, 89 insertions(+), 18 deletions(-) diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py index c54a094..c636e2f 100644 --- a/edkrepo/commands/sync_command.py +++ b/edkrepo/commands/sync_command.py @@ -22,9 +22,9 @@ from edkrepo.commands.edkrepo_command import EdkrepoCommand from edkrepo.commands.edkrepo_command import SubmoduleSkipArgument, SourceManifestRepoArgument import edkrepo.commands.arguments.sync_args as arguments -from edkrepo.common.edkrepo_exception import EdkrepoManifestNotFoundException +from edkrepo.common.edkrepo_exception import EdkrepoException, EdkrepoManifestNotFoundException from edkrepo.common.edkrepo_exception import EdkrepoManifestChangedException -from edkrepo.common.humble import SYNC_MANIFEST_NOT_FOUND +from edkrepo.common.humble import SYNC_MANIFEST_NOT_FOUND, SYNC_MANIFEST_UPDATE_FAILED from edkrepo.common.humble import SYNC_SOURCE_MOVE_WARNING, SYNC_REMOVE_WARNING, SYNC_REMOVE_LIST_END_FORMATTING from edkrepo.common.humble import SYNC_MANIFEST_DIFF_WARNING, SYNC_MANIFEST_UPDATE from edkrepo.common.humble import SPARSE_RESET, SPARSE_CHECKOUT, SYNC_REPO_CHANGE, SYNCING, FETCHING, UPDATING_MANIFEST @@ -35,7 +35,7 @@ from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import SOURCE_MANIFEST_REPO_NOT_FOUND from edkrepo.common.pathfix import get_actual_path, expanduser from edkrepo.common.common_cache_functions import get_repo_cache_obj -from edkrepo.common.common_repo_functions import clone_repos, sparse_checkout_enabled +from edkrepo.common.common_repo_functions import check_patchset_similarity, clone_repos, create_repos, patchset_application_flow, patchset_operations_similarity, sparse_checkout_enabled from edkrepo.common.common_repo_functions import reset_sparse_checkout, sparse_checkout, verify_single_manifest from edkrepo.common.common_repo_functions import checkout_repos, check_dirty_repos from edkrepo.common.common_repo_functions import update_editor_config @@ -131,7 +131,12 @@ def run_command(self, args, config): # Get the latest manifest if requested if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse - self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory) + try: + self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory) + except EdkrepoException as e: + if args.verbose: + print(e) + print(SYNC_MANIFEST_UPDATE_FAILED) manifest = get_workspace_manifest() if args.update_local_manifest: try: @@ -165,6 +170,8 @@ def run_command(self, args, config): if not args.update_local_manifest: #Performance optimization, __update_local_manifest() will do this self.__check_submodule_config(workspace_path, manifest, repo_sources_to_sync) clean_git_globalconfig() + manifest_repo = manifest.general_config.source_manifest_repo + global_manifest_path = get_manifest_repo_path(manifest_repo, config) for repo_to_sync in repo_sources_to_sync: local_repo_path = os.path.join(workspace_path, repo_to_sync.root) # Update any hooks @@ -173,7 +180,9 @@ def run_command(self, args, config): repo = Repo(local_repo_path) #Fetch notes repo.remotes.origin.fetch("refs/notes/*:refs/notes/*") - if repo_to_sync.commit is None and repo_to_sync.tag is None: + if repo_to_sync.patch_set: + patchset_application_flow(repo_to_sync, repo, workspace_path, manifest, global_manifest_path) + elif repo_to_sync.commit is None and repo_to_sync.tag is None: local_commits = False initial_active_branch = repo.active_branch repo.remotes.origin.fetch("refs/heads/{0}:refs/remotes/origin/{0}".format(repo_to_sync.branch)) @@ -287,8 +296,8 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path raise EdkrepoManifestNotFoundException(SYNC_MANIFEST_NOT_FOUND.format(initial_manifest.project_info.codename)) initial_manifest_remotes = {name:url for name, url in initial_manifest.remotes} ci_index_xml_rel_path = os.path.normpath(ci_index_xml.get_project_xml(initial_manifest.project_info.codename)) - global_manifest_path = os.path.join(global_manifest_directory, ci_index_xml_rel_path) - new_manifest_to_check = ManifestXml(global_manifest_path) + global_manifest = os.path.join(global_manifest_directory, ci_index_xml_rel_path) + new_manifest_to_check = ManifestXml(global_manifest) # Does the current combo exist in the new manifest? If not check to see if you can use the repo sources from # the default combo @@ -304,7 +313,7 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path write_included_config(new_manifest_to_check.remotes, new_manifest_to_check.submodule_alternate_remotes, local_manifest_dir) self.__check_submodule_config(workspace_path, new_manifest_to_check, new_sources_for_current_combo) - + manifest_path = get_manifest_repo_path(new_manifest_to_check.general_config.source_manifest_repo, config) # Check that the repo sources lists are the same. If they are not the same and the override flag is not set, throw an exception. if not args.override and set(initial_sources) != set(new_sources): raise EdkrepoManifestChangedException(SYNC_REPO_CHANGE.format(initial_manifest.project_info.codename)) @@ -388,7 +397,7 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path if len(sources_to_remove) > 0: ui_functions.print_warning_msg(SYNC_REMOVE_LIST_END_FORMATTING, header = False) # Clone any new Git repositories - clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, new_manifest_to_check) + clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, new_manifest_to_check, manifest_path) # Make a list of and only checkout repos that were newly cloned. Sync keeps repos on their initial active branches # cloning the entire combo can prevent existing repos from correctly being returned to their proper branch repos_to_checkout = [] @@ -397,15 +406,28 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path for source in sources_to_clone: if source.root == new_source.root: repos_to_checkout.append(source) - repos_to_checkout.extend(self.__check_combo_sha_tag_branch(workspace_path, initial_common, new_common)) + + new_repos_to_checkout, repos_to_create = self.__check_combo_patchset_sha_tag_branch(workspace_path, initial_common, new_common, initial_manifest, new_manifest_to_check) + repos_to_checkout.extend(new_repos_to_checkout) if repos_to_checkout: - checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check) + checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check, manifest_path) + + if repos_to_create: + create_repos(repos_to_create, workspace_path, new_manifest_to_check, manifest_path) + + if set(initial_sources) == set(new_sources): + repos_to_checkout, repos_to_create = self.__check_combo_patchset_sha_tag_branch(workspace_path, initial_sources, new_sources, initial_manifest, new_manifest_to_check) + if repos_to_checkout: + checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check, manifest_path) + + if repos_to_create: + create_repos(repos_to_create, workspace_path, new_manifest_to_check, manifest_path) #remove the old manifest file and copy the new one ui_functions.print_info_msg(UPDATING_MANIFEST, header = False) local_manifest_path = os.path.join(local_manifest_dir, 'Manifest.xml') os.remove(local_manifest_path) - shutil.copy(global_manifest_path, local_manifest_path) + shutil.copy(global_manifest, local_manifest_path) # Update the source manifest repository tag in the local copy of the manifest XML new_manifest = ManifestXml(local_manifest_path) @@ -417,17 +439,27 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path except EdkrepoManifestNotFoundException: pass - def __check_combo_sha_tag_branch(self, workspace_path, initial_sources, new_sources): + def __check_combo_patchset_sha_tag_branch(self, workspace_path, initial_sources, new_sources, initial_manifest, new_manifest_to_check): # Checks for changes in the defined SHAs, Tags or branches in the checked out combo. Returns # a list of repos to checkout. Checks to see if user is on appropriate SHA, tag or branch and # throws and exception if not. repos_to_checkout = [] + repos_to_create = [] for initial_source in initial_sources: for new_source in new_sources: if initial_source.root == new_source.root and initial_source.remote_name == new_source.remote_name and initial_source.remote_url == new_source.remote_url: local_repo_path = os.path.join(workspace_path, initial_source.root) repo = Repo(local_repo_path) - if initial_source.commit and initial_source.commit != new_source.commit: + if initial_source.patch_set: + initial_patchset = initial_manifest.get_patchset(initial_source.patch_set, initial_source.remote_name) + new_patchset = new_manifest_to_check.get_patchset(new_source.patch_set, new_source.remote_name) + if check_patchset_similarity(initial_patchset, new_patchset): + if not patchset_operations_similarity(initial_patchset, new_patchset, initial_manifest, new_manifest_to_check): + repos_to_create.append(new_source) + else: + repos_to_checkout.append(new_source) + break + elif initial_source.commit and initial_source.commit != new_source.commit: if repo.head.object.hexsha != initial_source.commit: ui_functions.print_info_msg(SYNC_BRANCH_CHANGE_ON_LOCAL.format(initial_source.branch, new_source.branch, initial_source.root), header = False) repos_to_checkout.append(new_source) @@ -443,7 +475,7 @@ def __check_combo_sha_tag_branch(self, workspace_path, initial_sources, new_sour ui_functions.print_info_msg(SYNC_BRANCH_CHANGE_ON_LOCAL.format(initial_source.branch, new_source.branch, initial_source.root), header = False) repos_to_checkout.append(new_source) break - return repos_to_checkout + return repos_to_checkout, repos_to_create def __check_for_new_manifest(self, args, config, initial_manifest, workspace_path, global_manifest_directory): #if the manifest repository for the current manifest was not found then there is no project with the manifest diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index 93d26ce..dd8c009 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -426,8 +426,7 @@ def patchset_application_flow(repo, repo_obj, workspace_path, manifest, global_m create_local_branch(repo.patch_set, patchset, global_manifest_path, manifest, repo_obj) def is_collision(json_path, patch_set, repo, global_manifest_path): - repo_name = repo.working_tree_dir - repo_name = repo_name.split("\\")[-1] + repo_name = os.path.basename(repo.working_dir) COLLISION = False for branch in repo.branches: if str(branch) == patch_set: @@ -455,6 +454,45 @@ def is_collision(json_path, patch_set, repo, global_manifest_path): return True return False +def patchset_operations_similarity(initial_patchset, new_patchset, initial_manifest, new_manifest): + return initial_manifest.get_patchset_operations(initial_patchset.name, initial_patchset.remote) \ + == new_manifest.get_patchset_operations(new_patchset.name, new_patchset.remote) + +def check_patchset_similarity(initial_patchset, new_patchset): + if initial_patchset.remote != new_patchset.remote: + return False + elif initial_patchset.name != new_patchset.name: + return False + elif initial_patchset.parent_sha != new_patchset.parent_sha: + return False + elif initial_patchset.fetch_branch != new_patchset.fetch_branch: + return False + + return True + +def create_repos(repos_to_create, workspace_path, manifest, global_manifest_path): + for repo_to_create in repos_to_create: + local_repo_path = os.path.join(workspace_path, repo_to_create.root) + repo = Repo(local_repo_path) + json_path = os.path.join(workspace_path, "repo") + json_path = os.path.join(json_path, "patchset_{}.json".format(os.path.basename(repo.working_dir))) + repo_name = os.path.basename(repo.working_dir) + patch_set = repo_to_create.patch_set + for branch in repo.branches: + if str(branch) == patch_set: + with open(json_path, 'r+') as f: + data = json.load(f) + patchset_data = data[repo_name] + for patch_data in patchset_data: + if patch_set in patch_data.values(): + patchset = manifest.get_patchset(repo_to_create.patch_set, repo_to_create.remote_name) + create_local_branch(patch_set, patchset, global_manifest_path, manifest, repo) + branch.rename(patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S")) + patch_data[patch_set] = patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S") + f.seek(0) + json.dump(data, f, indent=4) + break + def validate_manifest_repo(manifest_repo, verbose=False, archived=False): print(VERIFY_GLOBAL) if archived: diff --git a/edkrepo/common/humble.py b/edkrepo/common/humble.py index 6d076b9..3cf460c 100644 --- a/edkrepo/common/humble.py +++ b/edkrepo/common/humble.py @@ -51,6 +51,7 @@ SYNC_BRANCH_CHANGE_ON_LOCAL = 'The SHA, tag or branch defined in the current combo has changed from {} to {} for the {} repo.\n The current workspace is not on the SHA/tag/branch defined in the initial combo. Unable to checkout new SHA/tag/branch.\n' + SYNC_UPDATE_FIX SYNC_REBASE_CALC_FAIL = 'Unable to calculate if a rebase is required for the current branch' SYNC_INCOMPATIBLE_COMBO = 'No compatible combinations found in the latest manifest file. Cloning a new workspace is recommended. ' + SYNC_EXIT +SYNC_MANIFEST_UPDATE_FAILED = 'Failed to update manifest.' #informational messages for sync_command.py SYNCING = 'Syncing {0} to latest {1} branch ...' @@ -174,4 +175,4 @@ APPLYING_REVERT_FAILED = "Failed to revert to the commit: {}" APPLYING_CHERRY_PICK_FAILED = "Failed to cherry pick the commit: {}" REMOVE_REMOTE_FAILED = "Failed to remove the remote: {}" -CHECKING_OUT_DEFAULT = "Failed to apply one of the patchset operations. Checking out back to the default branch" \ No newline at end of file +CHECKING_OUT_DEFAULT = "Failed to apply one of the patchset operations. Checking out back to the default branch" From a1373a68b62a4fa34129712dc2c821611d90b7c9 Mon Sep 17 00:00:00 2001 From: Harsh Vora Date: Tue, 1 Nov 2022 17:58:24 -0700 Subject: [PATCH 19/23] Edkrepo: Creating local branches at checkout instead of clone - Creating local branches for other combos while checking them out. This ensures we are checking out correctly every time. - Fixed a bug which removes the cloning error. - Added -x flag to cherry-pick Signed-off-by: Harsh Vora Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/common/common_repo_functions.py | 42 ++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index dd8c009..527f338 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -78,7 +78,7 @@ def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest, global_manifest_path, cache_obj=None): - created_patch_sets = [] + # created_patch_sets = [] for repo_to_clone in repos_to_clone: local_repo_path = os.path.join(workspace_dir, repo_to_clone.root) local_repo_url = repo_to_clone.remote_url @@ -104,7 +104,7 @@ def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, # order of importance is 1)commit 2)tag 3)branch with only the higest priority being checked out if repo_to_clone.patch_set: patchset = manifest.get_patchset(repo_to_clone.patch_set, repo_to_clone.remote_name) - created_patch_sets.append((repo_to_clone.patch_set, repo_to_clone.remote_name)) + # created_patch_sets.append((repo_to_clone.patch_set, repo_to_clone.remote_name)) create_local_branch(repo_to_clone.patch_set, patchset, global_manifest_path, manifest, repo) elif repo_to_clone.commit: if args.verbose and (repo_to_clone.branch or repo_to_clone.tag): @@ -150,22 +150,22 @@ def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, # Create patch set branches - patchsets_in_manifest = manifest.get_patchsets_for_combo() - if patchsets_in_manifest: - default_combo_repo = None - - for combo, patch in patchsets_in_manifest.items(): - for repo_to_clone in manifest.get_repo_sources(combo): - if getattr(repo_to_clone, "patch_set"): - repo = Repo(os.path.join(workspace_dir, repo_to_clone.root)) - if (getattr(patch, "name"), getattr(repo_to_clone, "remote_name")) not in created_patch_sets: - patch = manifest.get_patchset(getattr(repo_to_clone, "patch_set"), getattr(repo_to_clone, "remote_name")) - create_local_branch(getattr(repo_to_clone, "patch_set"), patch, global_manifest_path, manifest, repo) - else: - default_combo_repo = repo - - if default_combo_repo is not None: - default_combo_repo.git.checkout(created_patch_sets[0][0]) + # patchsets_in_manifest = manifest.get_patchsets_for_combo() + # if patchsets_in_manifest: + # default_combo_repo = None + + # for combo, patch in patchsets_in_manifest.items(): + # for repo_to_clone in manifest.get_repo_sources(combo): + # if getattr(repo_to_clone, "patch_set"): + # repo = Repo(os.path.join(workspace_dir, repo_to_clone.root)) + # if (getattr(patch, "name"), getattr(repo_to_clone, "remote_name")) not in created_patch_sets: + # patch = manifest.get_patchset(getattr(repo_to_clone, "patch_set"), getattr(repo_to_clone, "remote_name")) + # create_local_branch(getattr(repo_to_clone, "patch_set"), patch, global_manifest_path, manifest, repo) + # else: + # default_combo_repo = repo + + # if default_combo_repo is not None: + # default_combo_repo.git.checkout(created_patch_sets[0][0]) def write_included_config(remotes, submodule_alt_remotes, repo_directory): included_configs = [] @@ -809,7 +809,7 @@ def create_local_branch(name, patchset, global_manifest_path, manifest_obj, repo path = os.path.join(repo_path, "repo") json_path = os.path.join(path, "patchset_{}.json".format(os.path.basename(repo.working_dir))) remote_list = manifest_obj.remotes - operations_list = manifest_obj.get_patchset_operations(patchset.name) + operations_list = manifest_obj.get_patchset_operations(patchset.name, patchset.remote) REMOTE_IN_REMOTE_LIST = False for remote in remote_list: if patchset.remote == remote.name: @@ -890,7 +890,7 @@ def apply_patchset_operations(repo, remote, operations_list, global_manifest_pat except: raise EdkrepoFetchBranchNotFoundException(FETCH_BRANCH_DOES_NOT_EXIST.format(operation.source_branch)) try: - repo.git.cherry_pick(operation.sha) + repo.git.execute(['git', 'cherry-pick', operation.sha, '-x']) except: raise EdkrepoCherryPickFailedException(APPLYING_CHERRY_PICK_FAILED.format(operation.sha)) try: @@ -905,6 +905,6 @@ def apply_patchset_operations(repo, remote, operations_list, global_manifest_pat except: raise EdkrepoFetchBranchNotFoundException(FETCH_BRANCH_DOES_NOT_EXIST.format(operation.source_branch)) try: - repo.git.cherry_pick(operation.sha) + repo.git.execute(['git', 'cherry-pick', operation.sha, '-x']) except: raise EdkrepoCherryPickFailedException(APPLYING_CHERRY_PICK_FAILED.format(operation.sha)) From 9102f8602973a9dda944f59f5e17223bcb95fc54 Mon Sep 17 00:00:00 2001 From: Harsh Vora Date: Mon, 14 Nov 2022 14:16:31 -0800 Subject: [PATCH 20/23] Edkrepo: Enable using Tags in parent_sha of patchsets - Checked whether the parent_sha is in the tag list. If yes, checkout tags/ otherwise normal checkout Signed-off-by: Harsh Vora harsh.vora@intel.com Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/common/common_repo_functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index 527f338..81aeaa0 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -822,7 +822,10 @@ def create_local_branch(name, patchset, global_manifest_path, manifest_obj, repo parent_patchset = manifest_obj.get_patchset(patchset.parent_sha, patchset.remote) repo.git.checkout(parent_patchset[2], b=name) except: - repo.git.checkout(patchset.parent_sha, b=name) + if patchset.parent_sha in repo.tags: + repo.git.checkout("tags/{}".format(patchset.parent_sha), b=name) + else: + repo.git.checkout(patchset.parent_sha, b=name) try: apply_patchset_operations(repo, remote, operations_list, global_manifest_path, remote_list) head_sha = repo.git.execute(['git', 'rev-parse', 'HEAD']) From 2fb50039655d6fb268bc8538d739d2de1be5c199 Mon Sep 17 00:00:00 2001 From: Harsh Vora Date: Tue, 29 Nov 2022 18:06:01 -0800 Subject: [PATCH 21/23] Edkrepo: Bug Fixes - Fixed the out of order parameters in clone_repos function. - Changed names of methods from patchset_application_flow and is_collision to patchset_branch_creation_flow and is_branch_name_collision respectively. - Removed the manifest_path variable in sync_command as it was calculating the wrong path leading to a bug. - Brought back the exception while creating a local branch for a branch name that already exists. - Removed magic strings from the code. - While checking for collision, used 'git rev-parse' on the branch name to get the Head instead of checking out to that branch. - Implemented __eq__ method for _Patchset class. - Removed the check_patchset_similarity method and used the equality operator instead. - When creating new repos at sync, changed the flow from creating and then renaming to renaming and then creating which would prevent a potential bug. - If applying a patch fails, running a 'git am --abort' to cancel the operation and then raise an exception. - While applying cherry pick from a different remote, we now check to see if the url in the manifest exists in the 'git remote -v' list. - If a Cherry pick fails, we check for a merge conflict and execute 'git cherry-pick --abort' if there is a conflict. - Removed ':' from the humble strings to make it consistent. Signed-off-by: Harsh Vora harsh.vora@intel.com Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/commands/clone_command.py | 2 +- edkrepo/commands/sync_command.py | 17 +++++---- edkrepo/common/common_repo_functions.py | 47 ++++++++++++------------- edkrepo/common/humble.py | 11 +++--- edkrepo_manifest_parser/edk_manifest.py | 6 ++++ 5 files changed, 43 insertions(+), 40 deletions(-) diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py index 5c0c20f..cdace26 100644 --- a/edkrepo/commands/clone_command.py +++ b/edkrepo/commands/clone_command.py @@ -177,7 +177,7 @@ def run_command(self, args, config): cache_obj = get_repo_cache_obj(config) if cache_obj is not None: add_missing_cache_repos(cache_obj, manifest, args.verbose) - clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest, cache_obj, manifest_repository_path) + clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest, manifest_repository_path, cache_obj) # Init submodules if not args.skip_submodule: diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py index c636e2f..30bff7c 100644 --- a/edkrepo/commands/sync_command.py +++ b/edkrepo/commands/sync_command.py @@ -35,7 +35,7 @@ from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import SOURCE_MANIFEST_REPO_NOT_FOUND from edkrepo.common.pathfix import get_actual_path, expanduser from edkrepo.common.common_cache_functions import get_repo_cache_obj -from edkrepo.common.common_repo_functions import check_patchset_similarity, clone_repos, create_repos, patchset_application_flow, patchset_operations_similarity, sparse_checkout_enabled +from edkrepo.common.common_repo_functions import clone_repos, create_repos, patchset_branch_creation_flow, patchset_operations_similarity, sparse_checkout_enabled from edkrepo.common.common_repo_functions import reset_sparse_checkout, sparse_checkout, verify_single_manifest from edkrepo.common.common_repo_functions import checkout_repos, check_dirty_repos from edkrepo.common.common_repo_functions import update_editor_config @@ -181,7 +181,7 @@ def run_command(self, args, config): #Fetch notes repo.remotes.origin.fetch("refs/notes/*:refs/notes/*") if repo_to_sync.patch_set: - patchset_application_flow(repo_to_sync, repo, workspace_path, manifest, global_manifest_path) + patchset_branch_creation_flow(repo_to_sync, repo, workspace_path, manifest, global_manifest_path) elif repo_to_sync.commit is None and repo_to_sync.tag is None: local_commits = False initial_active_branch = repo.active_branch @@ -313,7 +313,6 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path write_included_config(new_manifest_to_check.remotes, new_manifest_to_check.submodule_alternate_remotes, local_manifest_dir) self.__check_submodule_config(workspace_path, new_manifest_to_check, new_sources_for_current_combo) - manifest_path = get_manifest_repo_path(new_manifest_to_check.general_config.source_manifest_repo, config) # Check that the repo sources lists are the same. If they are not the same and the override flag is not set, throw an exception. if not args.override and set(initial_sources) != set(new_sources): raise EdkrepoManifestChangedException(SYNC_REPO_CHANGE.format(initial_manifest.project_info.codename)) @@ -397,7 +396,7 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path if len(sources_to_remove) > 0: ui_functions.print_warning_msg(SYNC_REMOVE_LIST_END_FORMATTING, header = False) # Clone any new Git repositories - clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, new_manifest_to_check, manifest_path) + clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, new_manifest_to_check, global_manifest_directory) # Make a list of and only checkout repos that were newly cloned. Sync keeps repos on their initial active branches # cloning the entire combo can prevent existing repos from correctly being returned to their proper branch repos_to_checkout = [] @@ -410,18 +409,18 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path new_repos_to_checkout, repos_to_create = self.__check_combo_patchset_sha_tag_branch(workspace_path, initial_common, new_common, initial_manifest, new_manifest_to_check) repos_to_checkout.extend(new_repos_to_checkout) if repos_to_checkout: - checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check, manifest_path) + checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check, global_manifest_directory) if repos_to_create: - create_repos(repos_to_create, workspace_path, new_manifest_to_check, manifest_path) + create_repos(repos_to_create, workspace_path, new_manifest_to_check, global_manifest_directory) if set(initial_sources) == set(new_sources): repos_to_checkout, repos_to_create = self.__check_combo_patchset_sha_tag_branch(workspace_path, initial_sources, new_sources, initial_manifest, new_manifest_to_check) if repos_to_checkout: - checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check, manifest_path) + checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check, global_manifest_directory) if repos_to_create: - create_repos(repos_to_create, workspace_path, new_manifest_to_check, manifest_path) + create_repos(repos_to_create, workspace_path, new_manifest_to_check, global_manifest_directory) #remove the old manifest file and copy the new one ui_functions.print_info_msg(UPDATING_MANIFEST, header = False) @@ -453,7 +452,7 @@ def __check_combo_patchset_sha_tag_branch(self, workspace_path, initial_sources, if initial_source.patch_set: initial_patchset = initial_manifest.get_patchset(initial_source.patch_set, initial_source.remote_name) new_patchset = new_manifest_to_check.get_patchset(new_source.patch_set, new_source.remote_name) - if check_patchset_similarity(initial_patchset, new_patchset): + if initial_patchset == new_patchset: if not patchset_operations_similarity(initial_patchset, new_patchset, initial_manifest, new_manifest_to_check): repos_to_create.append(new_source) else: diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index 81aeaa0..e18c50f 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -21,7 +21,7 @@ from git import Repo import colorama -from edkrepo.common.edkrepo_exception import EdkrepoRevertFailedException, EdkrepoCherryPickFailedException +from edkrepo.common.edkrepo_exception import EdkrepoBranchExistsException, EdkrepoRevertFailedException, EdkrepoCherryPickFailedException from edkrepo.common.edkrepo_exception import EdkrepoFetchBranchNotFoundException from edkrepo.common.edkrepo_exception import EdkrepoPatchNotFoundException, EdkrepoPatchFailedException from edkrepo.common.edkrepo_exception import EdkrepoRemoteNotFoundException, EdkrepoRemoteAddException, EdkrepoRemoteRemoveException @@ -29,7 +29,7 @@ from edkrepo.common.edkrepo_exception import EdkrepoUncommitedChangesException from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException from edkrepo.common.progress_handler import GitProgressHandler -from edkrepo.common.humble import APPLYING_CHERRY_PICK_FAILED, APPLYING_PATCH_FAILED, APPLYING_REVERT_FAILED, CHECKING_OUT_DEFAULT, CHECKING_OUT_PATCHSET +from edkrepo.common.humble import APPLYING_CHERRY_PICK_FAILED, APPLYING_PATCH_FAILED, APPLYING_REVERT_FAILED, BRANCH_EXISTS, CHECKING_OUT_DEFAULT, CHECKING_OUT_PATCHSET from edkrepo.common.humble import FETCH_BRANCH_DOES_NOT_EXIST, PATCHFILE_DOES_NOT_EXIST from edkrepo.common.humble import REMOTE_CREATION_FAILED, REMOTE_NOT_FOUND, REMOVE_REMOTE_FAILED from edkrepo.common.humble import MISSING_BRANCH_COMMIT @@ -75,7 +75,8 @@ CLEAR_LINE = '\x1b[K' DEFAULT_REMOTE_NAME = 'origin' PRIMARY_REMOTE_NAME = 'primary' - +PATCH = "Patch" +REVERT = "Revert" def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest, global_manifest_path, cache_obj=None): # created_patch_sets = [] @@ -382,7 +383,7 @@ def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifes # the source section for the manifest the order of priority is the followiwng 1)patchset 2)commit # 3) tag 4)branch with the highest priority attribute provided beinng checked out if repo_to_checkout.patch_set: - patchset_application_flow(repo_to_checkout, repo, workspace_path, manifest, global_manifest_path) + patchset_branch_creation_flow(repo_to_checkout, repo, workspace_path, manifest, global_manifest_path) else: if repo_to_checkout.commit: if verbose and (repo_to_checkout.branch or repo_to_checkout.tag): @@ -414,18 +415,20 @@ def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifes else: raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT) -def patchset_application_flow(repo, repo_obj, workspace_path, manifest, global_manifest_path): +def patchset_branch_creation_flow(repo, repo_obj, workspace_path, manifest, global_manifest_path): json_path = os.path.join(workspace_path, "repo") json_path = os.path.join(json_path, "patchset_{}.json".format(os.path.basename(repo_obj.working_dir))) patchset = manifest.get_patchset(repo.patch_set, repo.remote_name) if repo.patch_set in repo_obj.branches: - if is_collision(json_path, repo.patch_set, repo_obj, global_manifest_path): + if is_branch_name_collision(json_path, repo.patch_set, repo_obj, global_manifest_path): create_local_branch(repo.patch_set, patchset, global_manifest_path, manifest, repo_obj) + else: + repo_obj.git.checkout(repo.patch_set) else: create_local_branch(repo.patch_set, patchset, global_manifest_path, manifest, repo_obj) -def is_collision(json_path, patch_set, repo, global_manifest_path): +def is_branch_name_collision(json_path, patch_set, repo, global_manifest_path): repo_name = os.path.basename(repo.working_dir) COLLISION = False for branch in repo.branches: @@ -435,8 +438,7 @@ def is_collision(json_path, patch_set, repo, global_manifest_path): patchset_data = data[repo_name] for patchset in patchset_data: if patch_set in patchset.values(): - repo.git.checkout(patch_set) - head = repo.git.execute(['git', 'rev-parse', 'HEAD']) + head = repo.git.execute(['git', 'rev-parse', patch_set]) if patchset['head_sha'] != head: patchset['head_sha'] = head COLLISION = True @@ -458,18 +460,6 @@ def patchset_operations_similarity(initial_patchset, new_patchset, initial_manif return initial_manifest.get_patchset_operations(initial_patchset.name, initial_patchset.remote) \ == new_manifest.get_patchset_operations(new_patchset.name, new_patchset.remote) -def check_patchset_similarity(initial_patchset, new_patchset): - if initial_patchset.remote != new_patchset.remote: - return False - elif initial_patchset.name != new_patchset.name: - return False - elif initial_patchset.parent_sha != new_patchset.parent_sha: - return False - elif initial_patchset.fetch_branch != new_patchset.fetch_branch: - return False - - return True - def create_repos(repos_to_create, workspace_path, manifest, global_manifest_path): for repo_to_create in repos_to_create: local_repo_path = os.path.join(workspace_path, repo_to_create.root) @@ -486,9 +476,9 @@ def create_repos(repos_to_create, workspace_path, manifest, global_manifest_path for patch_data in patchset_data: if patch_set in patch_data.values(): patchset = manifest.get_patchset(repo_to_create.patch_set, repo_to_create.remote_name) - create_local_branch(patch_set, patchset, global_manifest_path, manifest, repo) branch.rename(patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S")) patch_data[patch_set] = patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S") + create_local_branch(patch_set, patchset, global_manifest_path, manifest, repo) f.seek(0) json.dump(data, f, indent=4) break @@ -861,19 +851,24 @@ def create_local_branch(name, patchset, global_manifest_path, manifest_obj, repo json.dump(data, f, indent=4) f.close() +def is_merge_conflict(repo): + status = repo.git.status(porcelain=True).split() + return True if 'UU' in status else False + def apply_patchset_operations(repo, remote, operations_list, global_manifest_path, remote_list): for operations in operations_list: for operation in operations: - if operation.type == "Patch": + if operation.type == PATCH: path = os.path.normpath(os.path.join(global_manifest_path, operation.file)) if os.path.isfile(path): try: repo.git.execute(['git', 'am', path, '--ignore-whitespace']) except: + repo.git.execute(['git', 'am', '--abort']) raise EdkrepoPatchFailedException(APPLYING_PATCH_FAILED.format(operation.file)) else: raise EdkrepoPatchNotFoundException(PATCHFILE_DOES_NOT_EXIST.format(operation.file)) - elif operation.type == "Revert": + elif operation.type == REVERT: try: repo.git.execute(['git', 'revert', operation.sha, '--no-edit']) except: @@ -882,7 +877,7 @@ def apply_patchset_operations(repo, remote, operations_list, global_manifest_pat if operation.source_remote: REMOTE_FOUND = False for remote in remote_list: - if operation.source_remote == remote.name: + if operation.source_remote == remote.name and remote.url in repo.git.execute(['git', 'remote', '-v']): REMOTE_FOUND = True try: repo.git.execute(['git', 'remote', 'add', operation.source_remote, remote.url]) @@ -895,6 +890,8 @@ def apply_patchset_operations(repo, remote, operations_list, global_manifest_pat try: repo.git.execute(['git', 'cherry-pick', operation.sha, '-x']) except: + if is_merge_conflict(repo): + repo.git.execute(['git', 'cherry-pick', '--abort']) raise EdkrepoCherryPickFailedException(APPLYING_CHERRY_PICK_FAILED.format(operation.sha)) try: repo.git.execute(['git', 'remote', 'remove', operation.source_remote]) diff --git a/edkrepo/common/humble.py b/edkrepo/common/humble.py index 3cf460c..13afb09 100644 --- a/edkrepo/common/humble.py +++ b/edkrepo/common/humble.py @@ -167,12 +167,13 @@ SUBMODULE_DEINIT_FAILED = 'Warning: Unable to remove all submodule content' # Creating Local Branch Error Messages -REMOTE_NOT_FOUND = "Could not find the remote: {}" -REMOTE_CREATION_FAILED = "Failed to add the remote: {}" +BRANCH_EXISTS = "The branch {} already exists." +REMOTE_NOT_FOUND = "Could not find the remote {}" +REMOTE_CREATION_FAILED = "Failed to add the remote {}" FETCH_BRANCH_DOES_NOT_EXIST = "The branch {} does not exist" PATCHFILE_DOES_NOT_EXIST = "The patch file {} does not exist" APPLYING_PATCH_FAILED = "Unable to apply the patch file {}" -APPLYING_REVERT_FAILED = "Failed to revert to the commit: {}" -APPLYING_CHERRY_PICK_FAILED = "Failed to cherry pick the commit: {}" -REMOVE_REMOTE_FAILED = "Failed to remove the remote: {}" +APPLYING_REVERT_FAILED = "Failed to revert to the commit {}" +APPLYING_CHERRY_PICK_FAILED = "Failed to cherry pick the commit {}" +REMOVE_REMOTE_FAILED = "Failed to remove the remote {}" CHECKING_OUT_DEFAULT = "Failed to apply one of the patchset operations. Checking out back to the default branch" diff --git a/edkrepo_manifest_parser/edk_manifest.py b/edkrepo_manifest_parser/edk_manifest.py index 78cad75..2b54cad 100644 --- a/edkrepo_manifest_parser/edk_manifest.py +++ b/edkrepo_manifest_parser/edk_manifest.py @@ -847,6 +847,12 @@ def __init__(self, element): except KeyError as k: raise KeyError(REQUIRED_ATTRIB_ERROR_MSG.format(k, element.tag)) + def __eq__(self, other_patchset): + if isinstance(other_patchset, _PatchSet): + return (self.name, self.remote, self.parentSha, self.fetchBranch) == \ + (other_patchset.name, other_patchset.remote, other_patchset.parentSha, other_patchset.fetchBranch) + return False + @property def tuple(self): return PatchSet(self.remote, self.name, self.parentSha, self.fetchBranch) From 8a6462fba487300475be56560ce603eecd93a818 Mon Sep 17 00:00:00 2001 From: Harsh Vora Date: Wed, 30 Nov 2022 16:28:04 -0800 Subject: [PATCH 22/23] Implemented checking changes to local manifest while checkout and sync - Added patchset information to json for reference. - Updated the implementation to store patch file info to use a list of files so multiple patch files for a single combo are supported, updated collision detection to accomodate those changes - Checking for changes in local_manifest while checking for collision - Updated the sync -u flow accordingly Signed-off-by: Harsh Vora harsh.vora@intel.com Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/commands/sync_command.py | 4 +- edkrepo/common/common_repo_functions.py | 80 ++++++++++++++----------- 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py index 30bff7c..c8ccfcb 100644 --- a/edkrepo/commands/sync_command.py +++ b/edkrepo/commands/sync_command.py @@ -455,8 +455,10 @@ def __check_combo_patchset_sha_tag_branch(self, workspace_path, initial_sources, if initial_patchset == new_patchset: if not patchset_operations_similarity(initial_patchset, new_patchset, initial_manifest, new_manifest_to_check): repos_to_create.append(new_source) - else: + break repos_to_checkout.append(new_source) + else: + repos_to_create.append(new_source) break elif initial_source.commit and initial_source.commit != new_source.commit: if repo.head.object.hexsha != initial_source.commit: diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index e18c50f..4223636 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -79,7 +79,6 @@ REVERT = "Revert" def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest, global_manifest_path, cache_obj=None): - # created_patch_sets = [] for repo_to_clone in repos_to_clone: local_repo_path = os.path.join(workspace_dir, repo_to_clone.root) local_repo_url = repo_to_clone.remote_url @@ -105,7 +104,6 @@ def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, # order of importance is 1)commit 2)tag 3)branch with only the higest priority being checked out if repo_to_clone.patch_set: patchset = manifest.get_patchset(repo_to_clone.patch_set, repo_to_clone.remote_name) - # created_patch_sets.append((repo_to_clone.patch_set, repo_to_clone.remote_name)) create_local_branch(repo_to_clone.patch_set, patchset, global_manifest_path, manifest, repo) elif repo_to_clone.commit: if args.verbose and (repo_to_clone.branch or repo_to_clone.tag): @@ -149,25 +147,6 @@ def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, # Add the commit template if it exists. update_repo_commit_template(workspace_dir, repo, repo_to_clone, global_manifest_directory) - - # Create patch set branches - # patchsets_in_manifest = manifest.get_patchsets_for_combo() - # if patchsets_in_manifest: - # default_combo_repo = None - - # for combo, patch in patchsets_in_manifest.items(): - # for repo_to_clone in manifest.get_repo_sources(combo): - # if getattr(repo_to_clone, "patch_set"): - # repo = Repo(os.path.join(workspace_dir, repo_to_clone.root)) - # if (getattr(patch, "name"), getattr(repo_to_clone, "remote_name")) not in created_patch_sets: - # patch = manifest.get_patchset(getattr(repo_to_clone, "patch_set"), getattr(repo_to_clone, "remote_name")) - # create_local_branch(getattr(repo_to_clone, "patch_set"), patch, global_manifest_path, manifest, repo) - # else: - # default_combo_repo = repo - - # if default_combo_repo is not None: - # default_combo_repo.git.checkout(created_patch_sets[0][0]) - def write_included_config(remotes, submodule_alt_remotes, repo_directory): included_configs = [] for remote in remotes: @@ -419,38 +398,59 @@ def patchset_branch_creation_flow(repo, repo_obj, workspace_path, manifest, glob json_path = os.path.join(workspace_path, "repo") json_path = os.path.join(json_path, "patchset_{}.json".format(os.path.basename(repo_obj.working_dir))) patchset = manifest.get_patchset(repo.patch_set, repo.remote_name) + operations = manifest.get_patchset_operations(patchset.name, patchset.remote) if repo.patch_set in repo_obj.branches: - if is_branch_name_collision(json_path, repo.patch_set, repo_obj, global_manifest_path): + if is_branch_name_collision(json_path, repo.patch_set, repo_obj, global_manifest_path, operations): create_local_branch(repo.patch_set, patchset, global_manifest_path, manifest, repo_obj) else: repo_obj.git.checkout(repo.patch_set) else: create_local_branch(repo.patch_set, patchset, global_manifest_path, manifest, repo_obj) -def is_branch_name_collision(json_path, patch_set, repo, global_manifest_path): +def is_branch_name_collision(json_path, patchset_obj, repo, global_manifest_path, operations): repo_name = os.path.basename(repo.working_dir) + patchset_name = patchset_obj.name COLLISION = False for branch in repo.branches: - if str(branch) == patch_set: + if str(branch) == patchset_name: with open(json_path, 'r+') as f: data = json.load(f) patchset_data = data[repo_name] for patchset in patchset_data: - if patch_set in patchset.values(): - head = repo.git.execute(['git', 'rev-parse', patch_set]) + if patchset_name in patchset.values(): + head = repo.git.execute(['git', 'rev-parse', patchset_name]) + # detect change in branch if patchset['head_sha'] != head: patchset['head_sha'] = head COLLISION = True - if len(patchset.keys()) == 3: - patch_file = list(patchset.keys())[-1] - hash_of_patch_file = get_hash_of_file(os.path.normpath(os.path.join(global_manifest_path, patch_file))) - if patchset[patch_file] != hash_of_patch_file: - patchset[patch_file] = hash_of_patch_file - COLLISION = True + + # detect change in patch file + if patchset['patch_file']: + for patch in patchset['patch_file']: + patch_file = patch['file_name'] + hash_of_patch_file = get_hash_of_file(os.path.normpath(os.path.join(global_manifest_path, patch_file))) + if patch['hash'] != hash_of_patch_file: + patch['hash'] = hash_of_patch_file + COLLISION = True + + # detect change in local manifest + if patchset['remote'] != patchset_obj.remote: + patchset['remote'] = patchset_obj.remote + COLLISION = True + if patchset['parent_sha'] != patchset_obj.parent_sha: + patchset['parent_sha'] = patchset_obj.parent_sha + COLLISION = True + if patchset['fetch_branch'] != patchset_obj.fetch_branch: + patchset['fetch_branch'] = patchset_obj.fetch_branch + COLLISION = True + if patchset['patchset_operations'] != operations: + patchset['patchset_operations'] = operations + COLLISION = True + if COLLISION: - branch.rename(patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S")) - patchset[patch_set] = patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S") + branch.rename(patchset_name + '_' + time.strftime("%Y/%m/%d_%H_%M_%S")) + patchset[patchset_name] = patchset_name + '_' + time.strftime("%Y/%m/%d_%H_%M_%S") f.seek(0) json.dump(data, f, indent=4) return True @@ -831,13 +831,21 @@ def create_local_branch(name, patchset, global_manifest_path, manifest_obj, repo json_str = { patchset.name: name, - "head_sha": head_sha + "head_sha": head_sha, + "remote": patchset.remote, + "parent_sha": patchset.parent_sha, + "fetch_branch": patchset.fetch_branch, + "patchset_operations": operations_list, + "patch_file": [] } for operations in operations_list: for operation in operations: if operation.type == "Patch": - json_str[operation.file] = get_hash_of_file(os.path.normpath(os.path.join(global_manifest_path, operation.file))) + json_str["patch_file"].append({ + "file_name": operation.file, + "hash": get_hash_of_file(os.path.normpath(os.path.join(global_manifest_path, operation.file))) + }) if not os.path.isfile(json_path): with open(json_path, 'w') as f: From ae8186c71349897a571b272393bb3b45ce368e88 Mon Sep 17 00:00:00 2001 From: Harsh Vora Date: Tue, 13 Dec 2022 02:47:06 -0800 Subject: [PATCH 23/23] Edkrepo: Bug Fixes and Improvements - Implemented Patchset naming restriction to not use "main" or "master" as names. - Fixed the bug of edkrepo sync -u not working because of two file pointers opening at the same time in create_repos and create_local_branch. - Fixed a bug of named_tuples not being assigned in json. - Implemented the functionality to take into account user created local branches of the same name as patchset and to override and not create branches using edkrepo if they want to use their own branches. If the override flag is not specified, it throws an exception to tell the user to manually resolve branch name conflicts. Signed-off-by: Harsh Vora harsh.vora@intel.com Reviewed-by: Ashley DeSimone Reviewed-by: Kevin Sun Reviewed-by: Nate DeSimone --- edkrepo/common/common_repo_functions.py | 79 +++++++++++++++---------- edkrepo/common/edkrepo_exception.py | 3 + edkrepo/common/humble.py | 2 + edkrepo_manifest_parser/edk_manifest.py | 7 ++- 4 files changed, 59 insertions(+), 32 deletions(-) diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index 4223636..c6eb2af 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -21,7 +21,7 @@ from git import Repo import colorama -from edkrepo.common.edkrepo_exception import EdkrepoBranchExistsException, EdkrepoRevertFailedException, EdkrepoCherryPickFailedException +from edkrepo.common.edkrepo_exception import EdkrepoBranchExistsException, EdkrepoException, EdkrepoLocalBranchExistsException, EdkrepoRevertFailedException, EdkrepoCherryPickFailedException from edkrepo.common.edkrepo_exception import EdkrepoFetchBranchNotFoundException from edkrepo.common.edkrepo_exception import EdkrepoPatchNotFoundException, EdkrepoPatchFailedException from edkrepo.common.edkrepo_exception import EdkrepoRemoteNotFoundException, EdkrepoRemoteAddException, EdkrepoRemoteRemoveException @@ -29,8 +29,8 @@ from edkrepo.common.edkrepo_exception import EdkrepoUncommitedChangesException from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException from edkrepo.common.progress_handler import GitProgressHandler -from edkrepo.common.humble import APPLYING_CHERRY_PICK_FAILED, APPLYING_PATCH_FAILED, APPLYING_REVERT_FAILED, BRANCH_EXISTS, CHECKING_OUT_DEFAULT, CHECKING_OUT_PATCHSET -from edkrepo.common.humble import FETCH_BRANCH_DOES_NOT_EXIST, PATCHFILE_DOES_NOT_EXIST +from edkrepo.common.humble import APPLYING_CHERRY_PICK_FAILED, APPLYING_PATCH_FAILED, APPLYING_REVERT_FAILED, BRANCH_EXISTS, CHECKING_OUT_DEFAULT, CHECKING_OUT_PATCHSET, LOCAL_BRANCH_EXISTS +from edkrepo.common.humble import FETCH_BRANCH_DOES_NOT_EXIST, PATCHFILE_DOES_NOT_EXIST, COLLISION_DETECTED from edkrepo.common.humble import REMOTE_CREATION_FAILED, REMOTE_NOT_FOUND, REMOVE_REMOTE_FAILED from edkrepo.common.humble import MISSING_BRANCH_COMMIT from edkrepo.common.humble import UNCOMMITED_CHANGES, CHECKOUT_UNCOMMITED_CHANGES @@ -362,7 +362,10 @@ def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifes # the source section for the manifest the order of priority is the followiwng 1)patchset 2)commit # 3) tag 4)branch with the highest priority attribute provided beinng checked out if repo_to_checkout.patch_set: - patchset_branch_creation_flow(repo_to_checkout, repo, workspace_path, manifest, global_manifest_path) + try: + patchset_branch_creation_flow(repo_to_checkout, repo, workspace_path, manifest, global_manifest_path, override) + except EdkrepoLocalBranchExistsException: + raise else: if repo_to_checkout.commit: if verbose and (repo_to_checkout.branch or repo_to_checkout.tag): @@ -394,24 +397,34 @@ def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifes else: raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT) -def patchset_branch_creation_flow(repo, repo_obj, workspace_path, manifest, global_manifest_path): +def patchset_branch_creation_flow(repo, repo_obj, workspace_path, manifest, global_manifest_path, override): json_path = os.path.join(workspace_path, "repo") json_path = os.path.join(json_path, "patchset_{}.json".format(os.path.basename(repo_obj.working_dir))) patchset = manifest.get_patchset(repo.patch_set, repo.remote_name) - operations = manifest.get_patchset_operations(patchset.name, patchset.remote) + operations_list = manifest.get_patchset_operations(patchset.name, patchset.remote) + ops = [] + for operations in operations_list: + for operation in operations: + ops.append(operation._asdict()) if repo.patch_set in repo_obj.branches: - if is_branch_name_collision(json_path, repo.patch_set, repo_obj, global_manifest_path, operations): + try: + COLLISION = is_branch_name_collision(json_path, patchset, repo_obj, global_manifest_path, ops, override) + except EdkrepoLocalBranchExistsException: + raise + if COLLISION: + ui_functions.print_info_msg(COLLISION_DETECTED.format(repo.patch_set)) create_local_branch(repo.patch_set, patchset, global_manifest_path, manifest, repo_obj) else: repo_obj.git.checkout(repo.patch_set) else: create_local_branch(repo.patch_set, patchset, global_manifest_path, manifest, repo_obj) -def is_branch_name_collision(json_path, patchset_obj, repo, global_manifest_path, operations): +def is_branch_name_collision(json_path, patchset_obj, repo, global_manifest_path, operations, override): repo_name = os.path.basename(repo.working_dir) patchset_name = patchset_obj.name COLLISION = False + BRANCH_IN_JSON = False for branch in repo.branches: if str(branch) == patchset_name: with open(json_path, 'r+') as f: @@ -419,10 +432,11 @@ def is_branch_name_collision(json_path, patchset_obj, repo, global_manifest_path patchset_data = data[repo_name] for patchset in patchset_data: if patchset_name in patchset.values(): - head = repo.git.execute(['git', 'rev-parse', patchset_name]) + BRANCH_IN_JSON = True + # detect change in branch + head = repo.git.execute(['git', 'rev-parse', patchset_name]) if patchset['head_sha'] != head: - patchset['head_sha'] = head COLLISION = True # detect change in patch file @@ -431,21 +445,11 @@ def is_branch_name_collision(json_path, patchset_obj, repo, global_manifest_path patch_file = patch['file_name'] hash_of_patch_file = get_hash_of_file(os.path.normpath(os.path.join(global_manifest_path, patch_file))) if patch['hash'] != hash_of_patch_file: - patch['hash'] = hash_of_patch_file COLLISION = True # detect change in local manifest - if patchset['remote'] != patchset_obj.remote: - patchset['remote'] = patchset_obj.remote - COLLISION = True - if patchset['parent_sha'] != patchset_obj.parent_sha: - patchset['parent_sha'] = patchset_obj.parent_sha - COLLISION = True - if patchset['fetch_branch'] != patchset_obj.fetch_branch: - patchset['fetch_branch'] = patchset_obj.fetch_branch - COLLISION = True - if patchset['patchset_operations'] != operations: - patchset['patchset_operations'] = operations + if patchset['remote'] != patchset_obj.remote or patchset['parent_sha'] != patchset_obj.parent_sha \ + or patchset['fetch_branch'] != patchset_obj.fetch_branch or operations != patchset['patchset_operations']: COLLISION = True if COLLISION: @@ -454,7 +458,13 @@ def is_branch_name_collision(json_path, patchset_obj, repo, global_manifest_path f.seek(0) json.dump(data, f, indent=4) return True - return False + if not BRANCH_IN_JSON: + if not override: + raise EdkrepoLocalBranchExistsException(LOCAL_BRANCH_EXISTS.format(patchset_name)) + else: + return False + else: + return False def patchset_operations_similarity(initial_patchset, new_patchset, initial_manifest, new_manifest): return initial_manifest.get_patchset_operations(initial_patchset.name, initial_patchset.remote) \ @@ -470,18 +480,21 @@ def create_repos(repos_to_create, workspace_path, manifest, global_manifest_path patch_set = repo_to_create.patch_set for branch in repo.branches: if str(branch) == patch_set: + COLLISION = False with open(json_path, 'r+') as f: data = json.load(f) patchset_data = data[repo_name] for patch_data in patchset_data: if patch_set in patch_data.values(): - patchset = manifest.get_patchset(repo_to_create.patch_set, repo_to_create.remote_name) branch.rename(patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S")) patch_data[patch_set] = patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S") - create_local_branch(patch_set, patchset, global_manifest_path, manifest, repo) f.seek(0) json.dump(data, f, indent=4) + COLLISION = True break + if COLLISION: + patchset = manifest.get_patchset(repo_to_create.patch_set, repo_to_create.remote_name) + create_local_branch(patch_set, patchset, global_manifest_path, manifest, repo) def validate_manifest_repo(manifest_repo, verbose=False, archived=False): print(VERIFY_GLOBAL) @@ -604,9 +617,10 @@ def checkout(combination, global_manifest_path, verbose=False, override=False, l # combination exists in the manifest if combination_is_in_manifest(combo, manifest): manifest.write_current_combo(combo) - except: + except EdkrepoException as e: if verbose: traceback.print_exc() + ui_functions.print_error_msg(e) print (CHECKOUT_COMBO_UNSUCCESSFULL.format(combo)) # Return to the initial combo, since there was an issue with cheking out the selected combo checkout_repos(verbose, override, initial_repo_sources, workspace_path, manifest, global_manifest_path) @@ -817,7 +831,7 @@ def create_local_branch(name, patchset, global_manifest_path, manifest_obj, repo else: repo.git.checkout(patchset.parent_sha, b=name) try: - apply_patchset_operations(repo, remote, operations_list, global_manifest_path, remote_list) + apply_patchset_operations(repo, operations_list, global_manifest_path, remote_list) head_sha = repo.git.execute(['git', 'rev-parse', 'HEAD']) except (EdkrepoPatchFailedException, EdkrepoRevertFailedException, git.GitCommandError, EdkrepoCherryPickFailedException) as exception: print(exception) @@ -829,13 +843,18 @@ def create_local_branch(name, patchset, global_manifest_path, manifest_obj, repo if not REMOTE_IN_REMOTE_LIST: raise EdkrepoRemoteNotFoundException(REMOTE_NOT_FOUND.format(patchset.remote)) + ops_list = [] + for operations in operations_list: + for operation in operations: + ops_list.append(operation._asdict()) + json_str = { patchset.name: name, "head_sha": head_sha, "remote": patchset.remote, "parent_sha": patchset.parent_sha, "fetch_branch": patchset.fetch_branch, - "patchset_operations": operations_list, + "patchset_operations": ops_list, "patch_file": [] } @@ -863,7 +882,7 @@ def is_merge_conflict(repo): status = repo.git.status(porcelain=True).split() return True if 'UU' in status else False -def apply_patchset_operations(repo, remote, operations_list, global_manifest_path, remote_list): +def apply_patchset_operations(repo, operations_list, global_manifest_path, remote_list): for operations in operations_list: for operation in operations: if operation.type == PATCH: @@ -885,7 +904,7 @@ def apply_patchset_operations(repo, remote, operations_list, global_manifest_pat if operation.source_remote: REMOTE_FOUND = False for remote in remote_list: - if operation.source_remote == remote.name and remote.url in repo.git.execute(['git', 'remote', '-v']): + if operation.source_remote == remote.name: REMOTE_FOUND = True try: repo.git.execute(['git', 'remote', 'add', operation.source_remote, remote.url]) diff --git a/edkrepo/common/edkrepo_exception.py b/edkrepo/common/edkrepo_exception.py index 9acd262..391b459 100644 --- a/edkrepo/common/edkrepo_exception.py +++ b/edkrepo/common/edkrepo_exception.py @@ -153,3 +153,6 @@ class EdkrepoPatchFailedException(EdkrepoException): def __init__(self, message): super().__init__(message, 136) +class EdkrepoLocalBranchExistsException(EdkrepoException): + def __init__(self, message): + super().__init__(message, 137) diff --git a/edkrepo/common/humble.py b/edkrepo/common/humble.py index 13afb09..8af1e31 100644 --- a/edkrepo/common/humble.py +++ b/edkrepo/common/humble.py @@ -177,3 +177,5 @@ APPLYING_CHERRY_PICK_FAILED = "Failed to cherry pick the commit {}" REMOVE_REMOTE_FAILED = "Failed to remove the remote {}" CHECKING_OUT_DEFAULT = "Failed to apply one of the patchset operations. Checking out back to the default branch" +LOCAL_BRANCH_EXISTS = "The branch {} already exists. Please resolve the branch name conflict before checking out again." +COLLISION_DETECTED = "A branch with the same name detected. Renaming the old {} branch and creating a new one from the manifest." diff --git a/edkrepo_manifest_parser/edk_manifest.py b/edkrepo_manifest_parser/edk_manifest.py index 2b54cad..d646b6b 100644 --- a/edkrepo_manifest_parser/edk_manifest.py +++ b/edkrepo_manifest_parser/edk_manifest.py @@ -57,9 +57,10 @@ UNSUPPORTED_TYPE_ERROR = "{} is not a supported xml type: {}" INVALID_XML_ERROR = "{} is not a valid xml file ({})" PATCHSET_UNKNOWN_ERROR = "Could not find a PatchSet named '{}' in '{}'" -REMOTE_DIFFERENT_ERROR = "The remote for patchset {}/{} is different from {}/{}" -NO_PATCHSET_IN_COMBO = "The Combination: {} does not have any patchsets." +REMOTE_DIFFERENT_ERROR = "The remote for Patchset {}/{} is different from {}/{}" +NO_PATCHSET_IN_COMBO = "The Combination: {} does not have any Patchsets." NO_PATCHSET_EXISTS = "The Patchset: {} does not exist" +INVALID_PATCHSET_NAME_ERROR = "The Patchset cannot be named: {}. Please rename the Patchset" class BaseXmlHelper(): def __init__(self, fileref, xml_types): @@ -341,6 +342,8 @@ def __init__(self, fileref): subroot = self._tree.find('PatchSets') if subroot is not None: for patchset in subroot.iter(tag='PatchSet'): + if patchset.attrib['name'] == "main" or patchset.attrib['name'] == "master": + raise ValueError(INVALID_PATCHSET_NAME_ERROR.format(patchset.attrib['name'])) self._patch_sets[(patchset.attrib['name'], getattr(_PatchSet(patchset).tuple, "remote"))] =_PatchSet(patchset).tuple operations = [] for subelem in patchset: