Skip to content

Commit

Permalink
OverrideValidation: Allow in-place updates of .inf files (microsoft#103)
Browse files Browse the repository at this point in the history 
Adds support to the command line interface tool to perform in-place
updates of .inf files. i.e. regenerate existing overrides in the same
file. Removes the usage of datetime.utcnow() as it is slated for
deprecation (microsoft#586),

Includes:
- microsoft#103
- microsoft#256
- microsoft#586

Co-authored-by: Antaeus Kleinert-Strand <[email protected]>
Co-authored-by: John Schock <[email protected]>
Co-authored-by: Aaron Pop <[email protected]>
  • Loading branch information
@joschock @apop5 @Javagedes
3 people authored and Javagedes committed Jun 24, 2024
1 parent 3514595 commit b0c8378
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 37 deletions.
143 changes: 106 additions & 37 deletions BaseTools/Plugin/OverrideValidation/OverrideValidation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@
##


import argparse
import hashlib
import io
import logging
import os
import io
import re
import sys
from datetime import datetime
import subprocess
import argparse
import hashlib
from datetime import datetime, timezone
from io import StringIO

#
Expand All @@ -24,11 +24,12 @@
#
#
try:
from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin
from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser
from edk2toollib.utility_functions import RunCmd
from edk2toolext.environment.plugintypes.uefi_build_plugin import \
IUefiBuildPlugin
from edk2toollib.uefi.edk2.parsers.dsc_parser import *
from edk2toollib.uefi.edk2.parsers.inf_parser import InfParser
from edk2toollib.uefi.edk2.path_utilities import Edk2Path
from edk2toollib.utility_functions import RunCmd

#Tuple for (version, entrycount)
FORMAT_VERSION_1 = (1, 4) #Version 1: #OVERRIDE : VERSION | PATH_TO_MODULE | HASH | YYYY-MM-DDThh-mm-ss
Expand Down Expand Up @@ -58,7 +59,7 @@ def GetErrStr (cls, errcode):
elif (errcode == cls.OR_INVALID_FORMAT):
str = 'INVALID_FORMAT'
elif (errcode == cls.OR_DSC_INF_NOT_FOUND):
str = 'DSC_FILE_NOT_FOUND'
str = 'FILE_NOT_FOUND'
elif (errcode == cls.OR_TARGET_INF_NOT_FOUND):
str = 'INF_FILE_NOT_FOUND'
else:
Expand Down Expand Up @@ -127,7 +128,7 @@ def override_plat_validate(self, thebuilder):
if m_result != self.OverrideResult.OR_ALL_GOOD:
if m_result != self.OverrideResult.OR_DSC_INF_NOT_FOUND:
result = m_result
logging.error("Override processing error %s in file %s" % (self.OverrideResult.GetErrStr(m_result), file))
logging.error("Override processing error %s in file/dir %s" % (self.OverrideResult.GetErrStr(m_result), file))

self.override_log_print(thebuilder, modulelist, status)

Expand All @@ -153,6 +154,10 @@ def override_detect_process(self, thebuilder, filepath, filelist, modulenode, st
if (list_path in filelist):
return result

# This processing step is only for files. If the filepath is a directory (meaning the directory is hashed), skip this step
if os.path.isdir(filepath):
return result

# Check for file existence, fail otherwise.
if not os.path.isfile(filepath):
return self.OverrideResult.OR_DSC_INF_NOT_FOUND
Expand Down Expand Up @@ -299,6 +304,7 @@ def override_process_line_with_version1(self, thebuilder, filelist, OverrideEntr
# Step 4: Parse the time of hash generation
try:
EntryTimestamp = datetime.strptime(OverrideEntry[3].strip(), "%Y-%m-%dT%H-%M-%S")
EntryTimestamp = EntryTimestamp.replace(tzinfo=timezone.utc)
except ValueError:
logging.error("Inf Override Parse Error, override parameter has invalid timestamp %s" %(OverrideEntry[3].strip()))
result = self.OverrideResult.OR_INVALID_FORMAT
Expand All @@ -313,7 +319,7 @@ def override_process_line_with_version1(self, thebuilder, filelist, OverrideEntr
# Step 6: House keeping
# Process the path to workspace/package path based add it to the parent node
overridden_rel_path = self.PathTool.GetEdk2RelativePathFromAbsolutePath(fullpath)
date_delta = datetime.utcnow() - EntryTimestamp
date_delta = datetime.now(timezone.utc) - EntryTimestamp

m_node.entry_hash = EntryHash
m_node.path = overridden_rel_path
Expand Down Expand Up @@ -396,7 +402,7 @@ def override_log_print(self, thebuilder, modulelist, status):
with open(logfile, 'w') as log:
log.write("Platform: %s\n" %(thebuilder.env.GetValue("PRODUCT_NAME")))
log.write("Version: %s\n" %(thebuilder.env.GetValue("BLD_*_BUILDID_STRING")))
log.write("Date: %s\n" %(datetime.utcnow().strftime("%Y-%m-%dT%H-%M-%S")))
log.write("Date: %s\n" %(datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S")))
log.write("Commit: %s\n" %(thebuilder.env.GetValue("BLD_*_BUILDSHA")))
log.write("State: %d/%d\n" %(status[0], status[1]))

Expand Down Expand Up @@ -572,6 +578,14 @@ def path_parse():
'-t', '--targetpath', dest = 'TargetPath', type=str,
help = '''Specify the absolute path to your target module/file/folder by passing t Path/To/Target or --targetpath Path/To/Target.'''
)
group.add_argument (
'-r', '--regenpath', dest = 'RegenPath', type=str,
help = '''Specify the absolute path to an inf with existing overrides to regen by passing r Path/To/Target or --regenpath Path/To/Target.'''
)
parser.add_argument (
'-p', '--packagepath', dest = 'RegenPackagePath', nargs="*", default=[],
help = '''Specify the packages path to be used to resolve relative paths when using --regenpath. ignored otherwise. Workspace is always included.'''
)
parser.add_argument (
'-v', '--version', dest = 'Version', default= 2, type=int,
help = '''This is the version of the override hash to produce (currently only 1 and 2 are valid)'''
Expand All @@ -594,19 +608,28 @@ def path_parse():
if not os.path.isfile(Paths.TargetPath):
raise RuntimeError("Module path is invalid.")


if Paths.Version < 1 or Paths.Version > len(FORMAT_VERSIONS):
raise RuntimeError("Version is invalid")

if not os.path.isdir(Paths.WorkSpace):
raise RuntimeError("Workspace path is invalid.")
if not os.path.isfile(Paths.TargetPath) and not os.path.isdir(Paths.TargetPath):
raise RuntimeError("Module path is invalid.")
# Needs to strip os.sep is to take care of the root path case
# For a folder, this will do nothing on a formatted abspath
# For a drive root, this will rip off the os.sep
if not os.path.normcase(Paths.TargetPath).startswith(os.path.normcase(Paths.WorkSpace.rstrip(os.sep)) + os.sep):
raise RuntimeError("Module is not within specified Workspace.")
if Paths.TargetPath is not None:
if not os.path.isfile(Paths.TargetPath) and not os.path.isdir(Paths.TargetPath):
raise RuntimeError("Module path is invalid.")
# Needs to strip os.sep is to take care of the root path case
# For a folder, this will do nothing on a formatted abspath
# For a drive root, this will rip off the os.sep
if not os.path.normcase(Paths.TargetPath).startswith(os.path.normcase(Paths.WorkSpace.rstrip(os.sep)) + os.sep):
raise RuntimeError("Module is not within specified Workspace.")

if Paths.RegenPath is not None:
if not os.path.isfile(Paths.RegenPath):
raise RuntimeError("Regen path is invalid.")
# Needs to strip os.sep is to take care of the root path case
# For a folder, this will do nothing on a formatted abspath
# For a drive root, this will rip off the os.sep
if not os.path.normcase(Paths.RegenPath).startswith(os.path.normcase(Paths.WorkSpace.rstrip(os.sep)) + os.sep):
raise RuntimeError("Module is not within specified Workspace.")

return Paths

Expand All @@ -623,23 +646,69 @@ def path_parse():
# Parse required paths passed from cmd line arguments
Paths = path_parse()

dummy_list = []
pathtool = Edk2Path(Paths.WorkSpace, dummy_list)

# Use absolute module path to find package path
pkg_path = pathtool.GetContainingPackage(Paths.TargetPath)
rel_path = Paths.TargetPath[Paths.TargetPath.find(pkg_path):]

rel_path = rel_path.replace('\\', '/')
mod_hash = ModuleHashCal(Paths.TargetPath)
# check if we are asked to update an .inf file "in-place"
if (Paths.RegenPath is not None):
pathtool = Edk2Path(Paths.WorkSpace, Paths.RegenPackagePath)

v1_regex = re.compile(r"#(Override|Track) : (.*?) \| (.*?) \| (.*?) \| (.*?)")
v2_regex = re.compile(r"#(Override|Track) : (.*?) \| (.*?) \| (.*?) \| (.*?) \| (.*?)")
with open (Paths.RegenPath) as fd:
RegenInfData = fd.read()

RegenInfOutData = ""
for line in RegenInfData.splitlines (True):
match = v1_regex.match(line)
if match is None:
match = v2_regex.match(line)

if match is not None:
rel_path = match.group(3)
abs_path = pathtool.GetAbsolutePathOnThisSystemFromEdk2RelativePath(rel_path)
if abs_path is not None:
mod_hash = ModuleHashCal(abs_path)
# only update the line if the hash has changed - this ensures the timestamp tracks actual changes rather than last time it was run.
if (mod_hash != match.group(4)):
VERSION_INDEX = Paths.Version - 1

if VERSION_INDEX == 0:
line = '#%s : %08d | %s | %s | %s\n' % (match.group(1), FORMAT_VERSION_1[0], rel_path, mod_hash, datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S"))
elif VERSION_INDEX == 1:
git_hash = ModuleGitHash(abs_path)
line = '#%s : %08d | %s | %s | %s | %s\n' % (match.group(1), FORMAT_VERSION_2[0], rel_path, mod_hash, datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S"), git_hash)
print("Updating:\n" + line)
else:
print(f"Warning: Could not resolve relative path {rel_path}. Override line not updated.\n")

VERSION_INDEX = Paths.Version - 1
RegenInfOutData += line

if VERSION_INDEX == 0:
print("Copy and paste the following line(s) to your overrider inf file(s):\n")
print('#%s : %08d | %s | %s | %s' % ("Override" if not Paths.Track else "Track", FORMAT_VERSION_1[0], rel_path, mod_hash, datetime.utcnow().strftime("%Y-%m-%dT%H-%M-%S")))
with open (Paths.RegenPath, "w") as fd:
fd.write(RegenInfOutData)

elif VERSION_INDEX == 1:
git_hash = ModuleGitHash(Paths.TargetPath)
print("Copy and paste the following line(s) to your overrider inf file(s):\n")
print('#%s : %08d | %s | %s | %s | %s' % ("Override" if not Paths.Track else "Track", FORMAT_VERSION_2[0], rel_path, mod_hash, datetime.utcnow().strftime("%Y-%m-%dT%H-%M-%S"), git_hash))
else:
dummy_list = []
pathtool = Edk2Path(Paths.WorkSpace, dummy_list)

# Generate and print the override for pasting into the file.
# Use absolute module path to find package path
pkg_path = pathtool.GetContainingPackage(Paths.TargetPath)
if pkg_path is not None:
rel_path = Paths.TargetPath[Paths.TargetPath.find(pkg_path):]
else:
rel_path = pathtool.GetEdk2RelativePathFromAbsolutePath(Paths.TargetPath)
if not rel_path:
print(f"{Paths.TargetPath} is invalid for this workspace.")
sys.exit(1)

rel_path = rel_path.replace('\\', '/')
mod_hash = ModuleHashCal(Paths.TargetPath)

VERSION_INDEX = Paths.Version - 1

if VERSION_INDEX == 0:
print("Copy and paste the following line(s) to your overrider inf file(s):\n")
print('#%s : %08d | %s | %s | %s' % ("Override" if not Paths.Track else "Track", FORMAT_VERSION_1[0], rel_path, mod_hash, datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S")))

elif VERSION_INDEX == 1:
git_hash = ModuleGitHash(Paths.TargetPath)
print("Copy and paste the following line(s) to your overrider inf file(s):\n")
print('#%s : %08d | %s | %s | %s | %s' % ("Override" if not Paths.Track else "Track", FORMAT_VERSION_2[0], rel_path, mod_hash, datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S"), git_hash))
21 changes: 21 additions & 0 deletions BaseTools/Plugin/OverrideValidation/ReadMe.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,27 @@ ORIGINALS:
```

Command to regenerate the override records in a given .inf file:

``` cmd
OverrideValidation.py -w c:\src -r c:\src\FooPkg\OverridingModule.inf
```

an example of the diff produced when using -r:

``` diff
diff --git a/FooPkg/OverridingModule.inf b/FooPkg/OverridingModule.inf
index 2d4ca47299..90da207a39 100644
--- a/FooPkg/OverridingModule.inf
+++ b/FooPkg/OverridingModule.inf
@@ -8,7 +8,7 @@
#
#
-#Override : 00000002 | BarPkg/OverridenModule.inf | 4f7eed98e3c084eecdff5fa2e1e57db1 | 2021-11-23T21-41-21 | 44b40c0358489da6c444e7cfb2be26e56b7c16a1
+#Override : 00000002 | BarPkg/OverridenModule.inf | 143b08782a2abc620d1eb57461c6e290 | 2022-03-10T23-09-45 | 6f8c53a3fcd79b202c708e7aa58256cafbf24bc4
#
```

## Versions

There are two versions of the override format.
Expand Down

0 comments on commit b0c8378

Please sign in to comment.