Skip to content

Commit

Permalink
Edkrepo: Integrated Sync Command with Creating Local Branch flow
Browse files Browse the repository at this point in the history 
- Implemented pathset similarity and patchset operation similarity
checking functions in common_repo_functions
- Made checks for the case where sources are same but patchset
operations have changed and created new branches for the new patchset
(using create_repos method implemented in common_repo_functions) while
preserving the old branch
- Did exception handling for updating the manifest because if there's
error while updating, the local copy of manifest should not change
- Bug fix of json not updated through create_repos
- Small bug fix in the is_collision method of common_repo_functions
- Bug Fix for global_manifest_path being different in sync file
- Name change of _check_sha_tag_branch to _check_patchset_sha_tag_branch

Signed-off-by: Harsh Vora [email protected]
Reviewed-by: Ashley DeSimone <[email protected]>
Reviewed-by:  Kevin Sun <[email protected]>
Reviewed-by: Nate DeSimone <[email protected]>
  • Loading branch information
@hrsh25 @ashedesimone
hrsh25 authored and ashedesimone committed Dec 20, 2022
1 parent e90b1e7 commit bf63945
Show file tree
Hide file tree
Showing 3 changed files with 89 additions and 18 deletions.
62 changes: 47 additions & 15 deletions edkrepo/commands/sync_command.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@
from edkrepo.commands.edkrepo_command import EdkrepoCommand
from edkrepo.commands.edkrepo_command import SubmoduleSkipArgument, SourceManifestRepoArgument
import edkrepo.commands.arguments.sync_args as arguments
from edkrepo.common.edkrepo_exception import EdkrepoManifestNotFoundException
from edkrepo.common.edkrepo_exception import EdkrepoException, EdkrepoManifestNotFoundException
from edkrepo.common.edkrepo_exception import EdkrepoManifestChangedException
from edkrepo.common.humble import SYNC_MANIFEST_NOT_FOUND
from edkrepo.common.humble import SYNC_MANIFEST_NOT_FOUND, SYNC_MANIFEST_UPDATE_FAILED
from edkrepo.common.humble import SYNC_SOURCE_MOVE_WARNING, SYNC_REMOVE_WARNING, SYNC_REMOVE_LIST_END_FORMATTING
from edkrepo.common.humble import SYNC_MANIFEST_DIFF_WARNING, SYNC_MANIFEST_UPDATE
from edkrepo.common.humble import SPARSE_RESET, SPARSE_CHECKOUT, SYNC_REPO_CHANGE, SYNCING, FETCHING, UPDATING_MANIFEST
Expand All @@ -35,7 +35,7 @@
from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import SOURCE_MANIFEST_REPO_NOT_FOUND
from edkrepo.common.pathfix import get_actual_path, expanduser
from edkrepo.common.common_cache_functions import get_repo_cache_obj
from edkrepo.common.common_repo_functions import clone_repos, sparse_checkout_enabled
from edkrepo.common.common_repo_functions import check_patchset_similarity, clone_repos, create_repos, patchset_application_flow, patchset_operations_similarity, sparse_checkout_enabled
from edkrepo.common.common_repo_functions import reset_sparse_checkout, sparse_checkout, verify_single_manifest
from edkrepo.common.common_repo_functions import checkout_repos, check_dirty_repos
from edkrepo.common.common_repo_functions import update_editor_config
Expand Down Expand Up @@ -131,7 +131,12 @@ def run_command(self, args, config):

# Get the latest manifest if requested
if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse
self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory)
try:
self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory)
except EdkrepoException as e:
if args.verbose:
print(e)
print(SYNC_MANIFEST_UPDATE_FAILED)
manifest = get_workspace_manifest()
if args.update_local_manifest:
try:
Expand Down Expand Up @@ -165,6 +170,8 @@ def run_command(self, args, config):
if not args.update_local_manifest: #Performance optimization, __update_local_manifest() will do this
self.__check_submodule_config(workspace_path, manifest, repo_sources_to_sync)
clean_git_globalconfig()
manifest_repo = manifest.general_config.source_manifest_repo
global_manifest_path = get_manifest_repo_path(manifest_repo, config)
for repo_to_sync in repo_sources_to_sync:
local_repo_path = os.path.join(workspace_path, repo_to_sync.root)
# Update any hooks
Expand All @@ -173,7 +180,9 @@ def run_command(self, args, config):
repo = Repo(local_repo_path)
#Fetch notes
repo.remotes.origin.fetch("refs/notes/*:refs/notes/*")
if repo_to_sync.commit is None and repo_to_sync.tag is None:
if repo_to_sync.patch_set:
patchset_application_flow(repo_to_sync, repo, workspace_path, manifest, global_manifest_path)
elif repo_to_sync.commit is None and repo_to_sync.tag is None:
local_commits = False
initial_active_branch = repo.active_branch
repo.remotes.origin.fetch("refs/heads/{0}:refs/remotes/origin/{0}".format(repo_to_sync.branch))
Expand Down Expand Up @@ -287,8 +296,8 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path
raise EdkrepoManifestNotFoundException(SYNC_MANIFEST_NOT_FOUND.format(initial_manifest.project_info.codename))
initial_manifest_remotes = {name:url for name, url in initial_manifest.remotes}
ci_index_xml_rel_path = os.path.normpath(ci_index_xml.get_project_xml(initial_manifest.project_info.codename))
global_manifest_path = os.path.join(global_manifest_directory, ci_index_xml_rel_path)
new_manifest_to_check = ManifestXml(global_manifest_path)
global_manifest = os.path.join(global_manifest_directory, ci_index_xml_rel_path)
new_manifest_to_check = ManifestXml(global_manifest)

# Does the current combo exist in the new manifest? If not check to see if you can use the repo sources from
# the default combo
Expand All @@ -304,7 +313,7 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path
write_included_config(new_manifest_to_check.remotes, new_manifest_to_check.submodule_alternate_remotes, local_manifest_dir)

self.__check_submodule_config(workspace_path, new_manifest_to_check, new_sources_for_current_combo)

manifest_path = get_manifest_repo_path(new_manifest_to_check.general_config.source_manifest_repo, config)
# Check that the repo sources lists are the same. If they are not the same and the override flag is not set, throw an exception.
if not args.override and set(initial_sources) != set(new_sources):
raise EdkrepoManifestChangedException(SYNC_REPO_CHANGE.format(initial_manifest.project_info.codename))
Expand Down Expand Up @@ -388,7 +397,7 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path
if len(sources_to_remove) > 0:
ui_functions.print_warning_msg(SYNC_REMOVE_LIST_END_FORMATTING, header = False)
# Clone any new Git repositories
clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, new_manifest_to_check)
clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, new_manifest_to_check, manifest_path)
# Make a list of and only checkout repos that were newly cloned. Sync keeps repos on their initial active branches
# cloning the entire combo can prevent existing repos from correctly being returned to their proper branch
repos_to_checkout = []
Expand All @@ -397,15 +406,28 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path
for source in sources_to_clone:
if source.root == new_source.root:
repos_to_checkout.append(source)
repos_to_checkout.extend(self.__check_combo_sha_tag_branch(workspace_path, initial_common, new_common))

new_repos_to_checkout, repos_to_create = self.__check_combo_patchset_sha_tag_branch(workspace_path, initial_common, new_common, initial_manifest, new_manifest_to_check)
repos_to_checkout.extend(new_repos_to_checkout)
if repos_to_checkout:
checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check)
checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check, manifest_path)

if repos_to_create:
create_repos(repos_to_create, workspace_path, new_manifest_to_check, manifest_path)

if set(initial_sources) == set(new_sources):
repos_to_checkout, repos_to_create = self.__check_combo_patchset_sha_tag_branch(workspace_path, initial_sources, new_sources, initial_manifest, new_manifest_to_check)
if repos_to_checkout:
checkout_repos(args.verbose, args.override, repos_to_checkout, workspace_path, new_manifest_to_check, manifest_path)

if repos_to_create:
create_repos(repos_to_create, workspace_path, new_manifest_to_check, manifest_path)

#remove the old manifest file and copy the new one
ui_functions.print_info_msg(UPDATING_MANIFEST, header = False)
local_manifest_path = os.path.join(local_manifest_dir, 'Manifest.xml')
os.remove(local_manifest_path)
shutil.copy(global_manifest_path, local_manifest_path)
shutil.copy(global_manifest, local_manifest_path)

# Update the source manifest repository tag in the local copy of the manifest XML
new_manifest = ManifestXml(local_manifest_path)
Expand All @@ -417,17 +439,27 @@ def __update_local_manifest(self, args, config, initial_manifest, workspace_path
except EdkrepoManifestNotFoundException:
pass

def __check_combo_sha_tag_branch(self, workspace_path, initial_sources, new_sources):
def __check_combo_patchset_sha_tag_branch(self, workspace_path, initial_sources, new_sources, initial_manifest, new_manifest_to_check):
# Checks for changes in the defined SHAs, Tags or branches in the checked out combo. Returns
# a list of repos to checkout. Checks to see if user is on appropriate SHA, tag or branch and
# throws and exception if not.
repos_to_checkout = []
repos_to_create = []
for initial_source in initial_sources:
for new_source in new_sources:
if initial_source.root == new_source.root and initial_source.remote_name == new_source.remote_name and initial_source.remote_url == new_source.remote_url:
local_repo_path = os.path.join(workspace_path, initial_source.root)
repo = Repo(local_repo_path)
if initial_source.commit and initial_source.commit != new_source.commit:
if initial_source.patch_set:
initial_patchset = initial_manifest.get_patchset(initial_source.patch_set, initial_source.remote_name)
new_patchset = new_manifest_to_check.get_patchset(new_source.patch_set, new_source.remote_name)
if check_patchset_similarity(initial_patchset, new_patchset):
if not patchset_operations_similarity(initial_patchset, new_patchset, initial_manifest, new_manifest_to_check):
repos_to_create.append(new_source)
else:
repos_to_checkout.append(new_source)
break
elif initial_source.commit and initial_source.commit != new_source.commit:
if repo.head.object.hexsha != initial_source.commit:
ui_functions.print_info_msg(SYNC_BRANCH_CHANGE_ON_LOCAL.format(initial_source.branch, new_source.branch, initial_source.root), header = False)
repos_to_checkout.append(new_source)
Expand All @@ -443,7 +475,7 @@ def __check_combo_sha_tag_branch(self, workspace_path, initial_sources, new_sour
ui_functions.print_info_msg(SYNC_BRANCH_CHANGE_ON_LOCAL.format(initial_source.branch, new_source.branch, initial_source.root), header = False)
repos_to_checkout.append(new_source)
break
return repos_to_checkout
return repos_to_checkout, repos_to_create

def __check_for_new_manifest(self, args, config, initial_manifest, workspace_path, global_manifest_directory):
#if the manifest repository for the current manifest was not found then there is no project with the manifest
Expand Down
42 changes: 40 additions & 2 deletions edkrepo/common/common_repo_functions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -426,8 +426,7 @@ def patchset_application_flow(repo, repo_obj, workspace_path, manifest, global_m
create_local_branch(repo.patch_set, patchset, global_manifest_path, manifest, repo_obj)

def is_collision(json_path, patch_set, repo, global_manifest_path):
repo_name = repo.working_tree_dir
repo_name = repo_name.split("\\")[-1]
repo_name = os.path.basename(repo.working_dir)
COLLISION = False
for branch in repo.branches:
if str(branch) == patch_set:
Expand Down Expand Up @@ -455,6 +454,45 @@ def is_collision(json_path, patch_set, repo, global_manifest_path):
return True
return False

def patchset_operations_similarity(initial_patchset, new_patchset, initial_manifest, new_manifest):
return initial_manifest.get_patchset_operations(initial_patchset.name, initial_patchset.remote) \
== new_manifest.get_patchset_operations(new_patchset.name, new_patchset.remote)

def check_patchset_similarity(initial_patchset, new_patchset):
if initial_patchset.remote != new_patchset.remote:
return False
elif initial_patchset.name != new_patchset.name:
return False
elif initial_patchset.parent_sha != new_patchset.parent_sha:
return False
elif initial_patchset.fetch_branch != new_patchset.fetch_branch:
return False

return True

def create_repos(repos_to_create, workspace_path, manifest, global_manifest_path):
for repo_to_create in repos_to_create:
local_repo_path = os.path.join(workspace_path, repo_to_create.root)
repo = Repo(local_repo_path)
json_path = os.path.join(workspace_path, "repo")
json_path = os.path.join(json_path, "patchset_{}.json".format(os.path.basename(repo.working_dir)))
repo_name = os.path.basename(repo.working_dir)
patch_set = repo_to_create.patch_set
for branch in repo.branches:
if str(branch) == patch_set:
with open(json_path, 'r+') as f:
data = json.load(f)
patchset_data = data[repo_name]
for patch_data in patchset_data:
if patch_set in patch_data.values():
patchset = manifest.get_patchset(repo_to_create.patch_set, repo_to_create.remote_name)
create_local_branch(patch_set, patchset, global_manifest_path, manifest, repo)
branch.rename(patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S"))
patch_data[patch_set] = patch_set + '_' + time.strftime("%Y/%m/%d_%H_%M_%S")
f.seek(0)
json.dump(data, f, indent=4)
break

def validate_manifest_repo(manifest_repo, verbose=False, archived=False):
print(VERIFY_GLOBAL)
if archived:
Expand Down
3 changes: 2 additions & 1 deletion edkrepo/common/humble.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
SYNC_BRANCH_CHANGE_ON_LOCAL = 'The SHA, tag or branch defined in the current combo has changed from {} to {} for the {} repo.\n The current workspace is not on the SHA/tag/branch defined in the initial combo. Unable to checkout new SHA/tag/branch.\n' + SYNC_UPDATE_FIX
SYNC_REBASE_CALC_FAIL = 'Unable to calculate if a rebase is required for the current branch'
SYNC_INCOMPATIBLE_COMBO = 'No compatible combinations found in the latest manifest file. Cloning a new workspace is recommended. ' + SYNC_EXIT
SYNC_MANIFEST_UPDATE_FAILED = 'Failed to update manifest.'

#informational messages for sync_command.py
SYNCING = 'Syncing {0} to latest {1} branch ...'
Expand Down Expand Up @@ -174,4 +175,4 @@
APPLYING_REVERT_FAILED = "Failed to revert to the commit: {}"
APPLYING_CHERRY_PICK_FAILED = "Failed to cherry pick the commit: {}"
REMOVE_REMOTE_FAILED = "Failed to remove the remote: {}"
CHECKING_OUT_DEFAULT = "Failed to apply one of the patchset operations. Checking out back to the default branch"
CHECKING_OUT_DEFAULT = "Failed to apply one of the patchset operations. Checking out back to the default branch"

0 comments on commit bf63945

Please sign in to comment.