From b0c83782f03fde17c046a2a619727ca5209230a5 Mon Sep 17 00:00:00 2001 From: John Schock Date: Thu, 21 Apr 2022 17:04:26 -0700 Subject: [PATCH] OverrideValidation: Allow in-place updates of .inf files (#103) Adds support to the command line interface tool to perform in-place updates of .inf files. i.e. regenerate existing overrides in the same file. Removes the usage of datetime.utcnow() as it is slated for deprecation (#586), Includes: - #103 - #256 - #586 Co-authored-by: Antaeus Kleinert-Strand Co-authored-by: John Schock Co-authored-by: Aaron Pop --- .../OverrideValidation/OverrideValidation.py | 143 +++++++++++++----- BaseTools/Plugin/OverrideValidation/ReadMe.md | 21 +++ 2 files changed, 127 insertions(+), 37 deletions(-) diff --git a/BaseTools/Plugin/OverrideValidation/OverrideValidation.py b/BaseTools/Plugin/OverrideValidation/OverrideValidation.py index 8568059a868..081a0320ede 100644 --- a/BaseTools/Plugin/OverrideValidation/OverrideValidation.py +++ b/BaseTools/Plugin/OverrideValidation/OverrideValidation.py @@ -8,14 +8,14 @@ ## +import argparse +import hashlib +import io import logging import os -import io +import re import sys -from datetime import datetime -import subprocess -import argparse -import hashlib +from datetime import datetime, timezone from io import StringIO # @@ -24,11 +24,12 @@ # # try: - from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin - from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser - from edk2toollib.utility_functions import RunCmd + from edk2toolext.environment.plugintypes.uefi_build_plugin import \ + IUefiBuildPlugin from edk2toollib.uefi.edk2.parsers.dsc_parser import * + from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser from edk2toollib.uefi.edk2.path_utilities import Edk2Path + from edk2toollib.utility_functions import RunCmd #Tuple for (version, entrycount) FORMAT_VERSION_1 = (1, 4) #Version 1: #OVERRIDE : VERSION | PATH_TO_MODULE | HASH | YYYY-MM-DDThh-mm-ss @@ -58,7 +59,7 @@ def GetErrStr (cls, errcode): elif (errcode == cls.OR_INVALID_FORMAT): str = 'INVALID_FORMAT' elif (errcode == cls.OR_DSC_INF_NOT_FOUND): - str = 'DSC_FILE_NOT_FOUND' + str = 'FILE_NOT_FOUND' elif (errcode == cls.OR_TARGET_INF_NOT_FOUND): str = 'INF_FILE_NOT_FOUND' else: @@ -127,7 +128,7 @@ def override_plat_validate(self, thebuilder): if m_result != self.OverrideResult.OR_ALL_GOOD: if m_result != self.OverrideResult.OR_DSC_INF_NOT_FOUND: result = m_result - logging.error("Override processing error %s in file %s" % (self.OverrideResult.GetErrStr(m_result), file)) + logging.error("Override processing error %s in file/dir %s" % (self.OverrideResult.GetErrStr(m_result), file)) self.override_log_print(thebuilder, modulelist, status) @@ -153,6 +154,10 @@ def override_detect_process(self, thebuilder, filepath, filelist, modulenode, st if (list_path in filelist): return result + # This processing step is only for files. If the filepath is a directory (meaning the directory is hashed), skip this step + if os.path.isdir(filepath): + return result + # Check for file existence, fail otherwise. if not os.path.isfile(filepath): return self.OverrideResult.OR_DSC_INF_NOT_FOUND @@ -299,6 +304,7 @@ def override_process_line_with_version1(self, thebuilder, filelist, OverrideEntr # Step 4: Parse the time of hash generation try: EntryTimestamp = datetime.strptime(OverrideEntry[3].strip(), "%Y-%m-%dT%H-%M-%S") + EntryTimestamp = EntryTimestamp.replace(tzinfo=timezone.utc) except ValueError: logging.error("Inf Override Parse Error, override parameter has invalid timestamp %s" %(OverrideEntry[3].strip())) result = self.OverrideResult.OR_INVALID_FORMAT @@ -313,7 +319,7 @@ def override_process_line_with_version1(self, thebuilder, filelist, OverrideEntr # Step 6: House keeping # Process the path to workspace/package path based add it to the parent node overridden_rel_path = self.PathTool.GetEdk2RelativePathFromAbsolutePath(fullpath) - date_delta = datetime.utcnow() - EntryTimestamp + date_delta = datetime.now(timezone.utc) - EntryTimestamp m_node.entry_hash = EntryHash m_node.path = overridden_rel_path @@ -396,7 +402,7 @@ def override_log_print(self, thebuilder, modulelist, status): with open(logfile, 'w') as log: log.write("Platform: %s\n" %(thebuilder.env.GetValue("PRODUCT_NAME"))) log.write("Version: %s\n" %(thebuilder.env.GetValue("BLD_*_BUILDID_STRING"))) - log.write("Date: %s\n" %(datetime.utcnow().strftime("%Y-%m-%dT%H-%M-%S"))) + log.write("Date: %s\n" %(datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S"))) log.write("Commit: %s\n" %(thebuilder.env.GetValue("BLD_*_BUILDSHA"))) log.write("State: %d/%d\n" %(status[0], status[1])) @@ -572,6 +578,14 @@ def path_parse(): '-t', '--targetpath', dest = 'TargetPath', type=str, help = '''Specify the absolute path to your target module/file/folder by passing t Path/To/Target or --targetpath Path/To/Target.''' ) + group.add_argument ( + '-r', '--regenpath', dest = 'RegenPath', type=str, + help = '''Specify the absolute path to an inf with existing overrides to regen by passing r Path/To/Target or --regenpath Path/To/Target.''' + ) + parser.add_argument ( + '-p', '--packagepath', dest = 'RegenPackagePath', nargs="*", default=[], + help = '''Specify the packages path to be used to resolve relative paths when using --regenpath. ignored otherwise. Workspace is always included.''' + ) parser.add_argument ( '-v', '--version', dest = 'Version', default= 2, type=int, help = '''This is the version of the override hash to produce (currently only 1 and 2 are valid)''' @@ -594,19 +608,28 @@ def path_parse(): if not os.path.isfile(Paths.TargetPath): raise RuntimeError("Module path is invalid.") - if Paths.Version < 1 or Paths.Version > len(FORMAT_VERSIONS): raise RuntimeError("Version is invalid") if not os.path.isdir(Paths.WorkSpace): raise RuntimeError("Workspace path is invalid.") - if not os.path.isfile(Paths.TargetPath) and not os.path.isdir(Paths.TargetPath): - raise RuntimeError("Module path is invalid.") - # Needs to strip os.sep is to take care of the root path case - # For a folder, this will do nothing on a formatted abspath - # For a drive root, this will rip off the os.sep - if not os.path.normcase(Paths.TargetPath).startswith(os.path.normcase(Paths.WorkSpace.rstrip(os.sep)) + os.sep): - raise RuntimeError("Module is not within specified Workspace.") + if Paths.TargetPath is not None: + if not os.path.isfile(Paths.TargetPath) and not os.path.isdir(Paths.TargetPath): + raise RuntimeError("Module path is invalid.") + # Needs to strip os.sep is to take care of the root path case + # For a folder, this will do nothing on a formatted abspath + # For a drive root, this will rip off the os.sep + if not os.path.normcase(Paths.TargetPath).startswith(os.path.normcase(Paths.WorkSpace.rstrip(os.sep)) + os.sep): + raise RuntimeError("Module is not within specified Workspace.") + + if Paths.RegenPath is not None: + if not os.path.isfile(Paths.RegenPath): + raise RuntimeError("Regen path is invalid.") + # Needs to strip os.sep is to take care of the root path case + # For a folder, this will do nothing on a formatted abspath + # For a drive root, this will rip off the os.sep + if not os.path.normcase(Paths.RegenPath).startswith(os.path.normcase(Paths.WorkSpace.rstrip(os.sep)) + os.sep): + raise RuntimeError("Module is not within specified Workspace.") return Paths @@ -623,23 +646,69 @@ def path_parse(): # Parse required paths passed from cmd line arguments Paths = path_parse() - dummy_list = [] - pathtool = Edk2Path(Paths.WorkSpace, dummy_list) - - # Use absolute module path to find package path - pkg_path = pathtool.GetContainingPackage(Paths.TargetPath) - rel_path = Paths.TargetPath[Paths.TargetPath.find(pkg_path):] - - rel_path = rel_path.replace('\\', '/') - mod_hash = ModuleHashCal(Paths.TargetPath) + # check if we are asked to update an .inf file "in-place" + if (Paths.RegenPath is not None): + pathtool = Edk2Path(Paths.WorkSpace, Paths.RegenPackagePath) + + v1_regex = re.compile(r"#(Override|Track) : (.*?) \| (.*?) \| (.*?) \| (.*?)") + v2_regex = re.compile(r"#(Override|Track) : (.*?) \| (.*?) \| (.*?) \| (.*?) \| (.*?)") + with open (Paths.RegenPath) as fd: + RegenInfData = fd.read() + + RegenInfOutData = "" + for line in RegenInfData.splitlines (True): + match = v1_regex.match(line) + if match is None: + match = v2_regex.match(line) + + if match is not None: + rel_path = match.group(3) + abs_path = pathtool.GetAbsolutePathOnThisSystemFromEdk2RelativePath(rel_path) + if abs_path is not None: + mod_hash = ModuleHashCal(abs_path) + # only update the line if the hash has changed - this ensures the timestamp tracks actual changes rather than last time it was run. + if (mod_hash != match.group(4)): + VERSION_INDEX = Paths.Version - 1 + + if VERSION_INDEX == 0: + line = '#%s : %08d | %s | %s | %s\n' % (match.group(1), FORMAT_VERSION_1[0], rel_path, mod_hash, datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S")) + elif VERSION_INDEX == 1: + git_hash = ModuleGitHash(abs_path) + line = '#%s : %08d | %s | %s | %s | %s\n' % (match.group(1), FORMAT_VERSION_2[0], rel_path, mod_hash, datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S"), git_hash) + print("Updating:\n" + line) + else: + print(f"Warning: Could not resolve relative path {rel_path}. Override line not updated.\n") - VERSION_INDEX = Paths.Version - 1 + RegenInfOutData += line - if VERSION_INDEX == 0: - print("Copy and paste the following line(s) to your overrider inf file(s):\n") - print('#%s : %08d | %s | %s | %s' % ("Override" if not Paths.Track else "Track", FORMAT_VERSION_1[0], rel_path, mod_hash, datetime.utcnow().strftime("%Y-%m-%dT%H-%M-%S"))) + with open (Paths.RegenPath, "w") as fd: + fd.write(RegenInfOutData) - elif VERSION_INDEX == 1: - git_hash = ModuleGitHash(Paths.TargetPath) - print("Copy and paste the following line(s) to your overrider inf file(s):\n") - print('#%s : %08d | %s | %s | %s | %s' % ("Override" if not Paths.Track else "Track", FORMAT_VERSION_2[0], rel_path, mod_hash, datetime.utcnow().strftime("%Y-%m-%dT%H-%M-%S"), git_hash)) + else: + dummy_list = [] + pathtool = Edk2Path(Paths.WorkSpace, dummy_list) + + # Generate and print the override for pasting into the file. + # Use absolute module path to find package path + pkg_path = pathtool.GetContainingPackage(Paths.TargetPath) + if pkg_path is not None: + rel_path = Paths.TargetPath[Paths.TargetPath.find(pkg_path):] + else: + rel_path = pathtool.GetEdk2RelativePathFromAbsolutePath(Paths.TargetPath) + if not rel_path: + print(f"{Paths.TargetPath} is invalid for this workspace.") + sys.exit(1) + + rel_path = rel_path.replace('\\', '/') + mod_hash = ModuleHashCal(Paths.TargetPath) + + VERSION_INDEX = Paths.Version - 1 + + if VERSION_INDEX == 0: + print("Copy and paste the following line(s) to your overrider inf file(s):\n") + print('#%s : %08d | %s | %s | %s' % ("Override" if not Paths.Track else "Track", FORMAT_VERSION_1[0], rel_path, mod_hash, datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S"))) + + elif VERSION_INDEX == 1: + git_hash = ModuleGitHash(Paths.TargetPath) + print("Copy and paste the following line(s) to your overrider inf file(s):\n") + print('#%s : %08d | %s | %s | %s | %s' % ("Override" if not Paths.Track else "Track", FORMAT_VERSION_2[0], rel_path, mod_hash, datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S"), git_hash)) diff --git a/BaseTools/Plugin/OverrideValidation/ReadMe.md b/BaseTools/Plugin/OverrideValidation/ReadMe.md index 17d1b199827..b3b9ac9e211 100644 --- a/BaseTools/Plugin/OverrideValidation/ReadMe.md +++ b/BaseTools/Plugin/OverrideValidation/ReadMe.md @@ -128,6 +128,27 @@ ORIGINALS: ``` +Command to regenerate the override records in a given .inf file: + +``` cmd +OverrideValidation.py -w c:\src -r c:\src\FooPkg\OverridingModule.inf +``` + +an example of the diff produced when using -r: + +``` diff +diff --git a/FooPkg/OverridingModule.inf b/FooPkg/OverridingModule.inf +index 2d4ca47299..90da207a39 100644 +--- a/FooPkg/OverridingModule.inf ++++ b/FooPkg/OverridingModule.inf +@@ -8,7 +8,7 @@ + # + # +-#Override : 00000002 | BarPkg/OverridenModule.inf | 4f7eed98e3c084eecdff5fa2e1e57db1 | 2021-11-23T21-41-21 | 44b40c0358489da6c444e7cfb2be26e56b7c16a1 ++#Override : 00000002 | BarPkg/OverridenModule.inf | 143b08782a2abc620d1eb57461c6e290 | 2022-03-10T23-09-45 | 6f8c53a3fcd79b202c708e7aa58256cafbf24bc4 + # +``` + ## Versions There are two versions of the override format.