diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py index c54a094..c636e2f 100644 --- a/edkrepo/commands/sync_command.py +++ b/edkrepo/commands/sync_command.py @@ -22,9 +22,9 @@ from edkrepo.commands.edkrepo_command import EdkrepoCommand from edkrepo.commands.edkrepo_command import SubmoduleSkipArgument, SourceManifestRepoArgument import edkrepo.commands.arguments.sync_args as arguments -from edkrepo.common.edkrepo_exception import EdkrepoManifestNotFoundException +from edkrepo.common.edkrepo_exception import EdkrepoException, EdkrepoManifestNotFoundException from edkrepo.common.edkrepo_exception import EdkrepoManifestChangedException -from edkrepo.common.humble import SYNC_MANIFEST_NOT_FOUND +from edkrepo.common.humble import SYNC_MANIFEST_NOT_FOUND, SYNC_MANIFEST_UPDATE_FAILED from edkrepo.common.humble import SYNC_SOURCE_MOVE_WARNING, SYNC_REMOVE_WARNING, SYNC_REMOVE_LIST_END_FORMATTING from edkrepo.common.humble import SYNC_MANIFEST_DIFF_WARNING, SYNC_MANIFEST_UPDATE from edkrepo.common.humble import SPARSE_RESET, SPARSE_CHECKOUT, SYNC_REPO_CHANGE, SYNCING, FETCHING, UPDATING_MANIFEST @@ -35,7 +35,7 @@ from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import SOURCE_MANIFEST_REPO_NOT_FOUND from edkrepo.common.pathfix import get_actual_path, expanduser from edkrepo.common.common_cache_functions import get_repo_cache_obj -from edkrepo.common.common_repo_functions import clone_repos, sparse_checkout_enabled +from edkrepo.common.common_repo_functions import check_patchset_similarity, clone_repos, create_repos, patchset_application_flow, patchset_operations_similarity, sparse_checkout_enabled from edkrepo.common.common_repo_functions import reset_sparse_checkout, sparse_checkout, verify_single_manifest from edkrepo.common.common_repo_functions import checkout_repos, check_dirty_repos from edkrepo.common.common_repo_functions import update_editor_config @@ -131,7 +131,12 @@ def run_command(self, args, config): # Get the latest manifest if requested if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse - self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory) + try: + self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory) + except EdkrepoException as e: + if args.verbose: + print(e) + print(SYNC_MANIFEST_UPDATE_FAILED) manifest = get_workspace_manifest() if args.update_local_manifest: try: @@ -165,6 +170,8 @@ def run_command(self, args, config): if not args.update_local_manifest: #Performance optimization, __update_local_manifest() will do this self.__check_submodule_config(workspace_path, manifest, repo_sources_to_sync) clean_git_globalconfig() + manifest_repo = manifest.general_config.source_manifest_repo + global_manifest_path = get_manifest_repo_path(manifest_repo, config) for repo_to_sync in repo_sources_to_sync: local_repo_path = os.path.join(workspace_path, repo_to_sync.root) # Update any hooks @@ -173,7 +180,9 @@ def run_command(self, args, config): repo = Repo(local_repo_path) #Fetch notes repo.remotes.origin.fetch("refs/notes/*:refs/notes/*") - if repo_to_sync.commit is None and repo_to_sync.tag is None: + if repo_to_sync.patch_set: + patchset_application_flow(repo_to_sync, repo, workspace_path, manifest, global_manifest_path) + elif repo_to_sync.commit is None and repo_to_sync.tag is None: local_commits = False initial_active_branch = repo.active_branch repo.remotes.origin.fetch("refs/heads/{0}:refs/remotes/origin/{0}".format(repo_to_sync.branch)) @@ -287,8 +296,8 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path raise EdkrepoManifestNotFoundException(SYNC_MANIFEST_NOT_FOUND.format(initial_manifest.project_info.codename)) initial_manifest_remotes = {name:url for name, url in initial_manifest.remotes} ci_index_xml_rel_path = os.path.normpath(ci_index_xml.get_project_xml(initial_manifest.project_info.codename)) - global_manifest_path = os.path.join(global_manifest_directory, ci_index_xml_rel_path) - new_manifest_to_check = ManifestXml(global_manifest_path) + global_manifest = os.path.join(global_manifest_directory, ci_index_xml_rel_path) + new_manifest_to_check = ManifestXml(global_manifest) # Does the current combo exist in the new manifest? If not check to see if you can use the repo sources from # the default combo @@ -304,7 +313,7 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path write_included_config(new_manifest_to_check.remotes, new_manifest_to_check.submodule_alternate_remotes, local_manifest_dir) self.__check_submodule_config(workspace_path, new_manifest_to_check, new_sources_for_current_combo) - + manifest_path = get_manifest_repo_path(new_manifest_to_check.general_config.source_manifest_repo, config) # Check that the repo sources lists are the same. If they are not the same and the override flag is not set, throw an exception. if not args.override and set(initial_sources) != set(new_sources): raise EdkrepoManifestChangedException(SYNC_REPO_CHANGE.format(initial_manifest.project_info.codename)) @@ -388,7 +397,7 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path if len(sources_to_remove) > 0: ui_functions.print_warning_msg(SYNC_REMOVE_LIST_END_FORMATTING, header = False) # Clone any new Git repositories - clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, new_manifest_to_check) + clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, new_manifest_to_check, manifest_path) # Make a list of and only checkout repos that were newly cloned. Sync keeps repos on their initial active branches # cloning the entire combo can prevent existing repos from correctly being returned to their proper branch repos_to_checkout = [] @@ -397,15 +406,28 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path for source in sources_to_clone: if source.root == new_source.root: repos_to_checkout.append(source) - repos_to_checkout.extend(self.__check_combo_sha_tag_branch(workspace_path, initial_common, new_common)) + + new_repos_to_checkout, repos_to_create = self.__check_combo_patchset_sha_tag_branch(workspace_path, initial_common, new_common, initial_manifest, new_manifest_to_check) + repos_to_checkout.extend(new_repos_to_checkout) if repos_to_checkout: - checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check) + checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check, manifest_path) + + if repos_to_create: + create_repos(repos_to_create, workspace_path, new_manifest_to_check, manifest_path) + + if set(initial_sources) == set(new_sources): + repos_to_checkout, repos_to_create = self.__check_combo_patchset_sha_tag_branch(workspace_path, initial_sources, new_sources, initial_manifest, new_manifest_to_check) + if repos_to_checkout: + checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check, manifest_path) + + if repos_to_create: + create_repos(repos_to_create, workspace_path, new_manifest_to_check, manifest_path) #remove the old manifest file and copy the new one ui_functions.print_info_msg(UPDATING_MANIFEST, header = False) local_manifest_path = os.path.join(local_manifest_dir, 'Manifest.xml') os.remove(local_manifest_path) - shutil.copy(global_manifest_path, local_manifest_path) + shutil.copy(global_manifest, local_manifest_path) # Update the source manifest repository tag in the local copy of the manifest XML new_manifest = ManifestXml(local_manifest_path) @@ -417,17 +439,27 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path except EdkrepoManifestNotFoundException: pass - def __check_combo_sha_tag_branch(self, workspace_path, initial_sources, new_sources): + def __check_combo_patchset_sha_tag_branch(self, workspace_path, initial_sources, new_sources, initial_manifest, new_manifest_to_check): # Checks for changes in the defined SHAs, Tags or branches in the checked out combo. Returns # a list of repos to checkout. Checks to see if user is on appropriate SHA, tag or branch and # throws and exception if not. repos_to_checkout = [] + repos_to_create = [] for initial_source in initial_sources: for new_source in new_sources: if initial_source.root == new_source.root and initial_source.remote_name == new_source.remote_name and initial_source.remote_url == new_source.remote_url: local_repo_path = os.path.join(workspace_path, initial_source.root) repo = Repo(local_repo_path) - if initial_source.commit and initial_source.commit != new_source.commit: + if initial_source.patch_set: + initial_patchset = initial_manifest.get_patchset(initial_source.patch_set, initial_source.remote_name) + new_patchset = new_manifest_to_check.get_patchset(new_source.patch_set, new_source.remote_name) + if check_patchset_similarity(initial_patchset, new_patchset): + if not patchset_operations_similarity(initial_patchset, new_patchset, initial_manifest, new_manifest_to_check): + repos_to_create.append(new_source) + else: + repos_to_checkout.append(new_source) + break + elif initial_source.commit and initial_source.commit != new_source.commit: if repo.head.object.hexsha != initial_source.commit: ui_functions.print_info_msg(SYNC_BRANCH_CHANGE_ON_LOCAL.format(initial_source.branch, new_source.branch, initial_source.root), header = False) repos_to_checkout.append(new_source) @@ -443,7 +475,7 @@ def __check_combo_sha_tag_branch(self, workspace_path, initial_sources, new_sour ui_functions.print_info_msg(SYNC_BRANCH_CHANGE_ON_LOCAL.format(initial_source.branch, new_source.branch, initial_source.root), header = False) repos_to_checkout.append(new_source) break - return repos_to_checkout + return repos_to_checkout, repos_to_create def __check_for_new_manifest(self, args, config, initial_manifest, workspace_path, global_manifest_directory): #if the manifest repository for the current manifest was not found then there is no project with the manifest diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py index 93d26ce..dd8c009 100644 --- a/edkrepo/common/common_repo_functions.py +++ b/edkrepo/common/common_repo_functions.py @@ -426,8 +426,7 @@ def patchset_application_flow(repo, repo_obj, workspace_path, manifest, global_m create_local_branch(repo.patch_set, patchset, global_manifest_path, manifest, repo_obj) def is_collision(json_path, patch_set, repo, global_manifest_path): - repo_name = repo.working_tree_dir - repo_name = repo_name.split("\\")[-1] + repo_name = os.path.basename(repo.working_dir) COLLISION = False for branch in repo.branches: if str(branch) == patch_set: @@ -455,6 +454,45 @@ def is_collision(json_path, patch_set, repo, global_manifest_path): return True return False +def patchset_operations_similarity(initial_patchset, new_patchset, initial_manifest, new_manifest): + return initial_manifest.get_patchset_operations(initial_patchset.name, initial_patchset.remote) \ + == new_manifest.get_patchset_operations(new_patchset.name, new_patchset.remote) + +def check_patchset_similarity(initial_patchset, new_patchset): + if initial_patchset.remote != new_patchset.remote: + return False + elif initial_patchset.name != new_patchset.name: + return False + elif initial_patchset.parent_sha != new_patchset.parent_sha: + return False + elif initial_patchset.fetch_branch != new_patchset.fetch_branch: + return False + + return True + +def create_repos(repos_to_create, workspace_path, manifest, global_manifest_path): + for repo_to_create in repos_to_create: + local_repo_path = os.path.join(workspace_path, repo_to_create.root) + repo = Repo(local_repo_path) + json_path = os.path.join(workspace_path, "repo") + json_path = os.path.join(json_path, "patchset_{}.json".format(os.path.basename(repo.working_dir))) + repo_name = os.path.basename(repo.working_dir) + patch_set = repo_to_create.patch_set + for branch in repo.branches: + if str(branch) == patch_set: + with open(json_path, 'r+') as f: + data = json.load(f) + patchset_data = data[repo_name] + for patch_data in patchset_data: + if patch_set in patch_data.values(): + patchset = manifest.get_patchset(repo_to_create.patch_set, repo_to_create.remote_name) + create_local_branch(patch_set, patchset, global_manifest_path, manifest, repo) + branch.rename(patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S")) + patch_data[patch_set] = patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S") + f.seek(0) + json.dump(data, f, indent=4) + break + def validate_manifest_repo(manifest_repo, verbose=False, archived=False): print(VERIFY_GLOBAL) if archived: diff --git a/edkrepo/common/humble.py b/edkrepo/common/humble.py index 6d076b9..3cf460c 100644 --- a/edkrepo/common/humble.py +++ b/edkrepo/common/humble.py @@ -51,6 +51,7 @@ SYNC_BRANCH_CHANGE_ON_LOCAL = 'The SHA, tag or branch defined in the current combo has changed from {} to {} for the {} repo.\n The current workspace is not on the SHA/tag/branch defined in the initial combo. Unable to checkout new SHA/tag/branch.\n' + SYNC_UPDATE_FIX SYNC_REBASE_CALC_FAIL = 'Unable to calculate if a rebase is required for the current branch' SYNC_INCOMPATIBLE_COMBO = 'No compatible combinations found in the latest manifest file. Cloning a new workspace is recommended. ' + SYNC_EXIT +SYNC_MANIFEST_UPDATE_FAILED = 'Failed to update manifest.' #informational messages for sync_command.py SYNCING = 'Syncing {0} to latest {1} branch ...' @@ -174,4 +175,4 @@ APPLYING_REVERT_FAILED = "Failed to revert to the commit: {}" APPLYING_CHERRY_PICK_FAILED = "Failed to cherry pick the commit: {}" REMOVE_REMOTE_FAILED = "Failed to remove the remote: {}" -CHECKING_OUT_DEFAULT = "Failed to apply one of the patchset operations. Checking out back to the default branch" \ No newline at end of file +CHECKING_OUT_DEFAULT = "Failed to apply one of the patchset operations. Checking out back to the default branch"