diff --git a/.azuredevops/pipelines/template-build.yml b/.azuredevops/pipelines/template-build.yml index 66a5955c..13121014 100644 --- a/.azuredevops/pipelines/template-build.yml +++ b/.azuredevops/pipelines/template-build.yml @@ -87,23 +87,14 @@ jobs: -f ${{ item.Value.Flavor }} -a ${{ item.Value.ArchList }} -t ${{ parameters.build_targets }} + --bundle --stop-on-fail TOOL_CHAIN_TAG=${{ item.Value.ToolChain }} - - task: CopyFiles@2 - displayName: Filter Driver Binaries # To reduce network consumption. - inputs: - sourceFolder: 'Build/CryptoBin_${{ item.Value.Flavor }}' - contents: | - **/*.efi - **/*.depex - **/BUILD_REPORT.TXT - targetFolder: '$(Build.ArtifactStagingDirectory)/Drivers' - flattenFolders: false - task: PublishPipelineArtifact@1 - displayName: Publish Driver Binaries + displayName: Publish Bundle inputs: - targetPath: '$(Build.ArtifactStagingDirectory)/Drivers' + targetPath: 'Bundle' artifactName: CryptoBin_${{ Item.Value.ArchList }}_${{ item.Value.Flavor }} - ${{ if eq(item.Value.CopyExtra, true) }}: @@ -161,70 +152,42 @@ jobs: patterns: 'CryptoBin_*/**' path: '$(Pipeline.Workspace)/Staging' - - task: DownloadPipelineArtifact@2 - displayName: Download Logs Artifacts - inputs: - patterns: '*_Logs/**' - path: '$(Pipeline.Workspace)/Logs' - - task: DownloadPipelineArtifact@2 displayName: Download Extras Artifacts inputs: artifact: Package_Extras path: '$(Pipeline.Workspace)/Extras' + - task: PythonScript@0 + displayName: Merge CryptoBin Artifacts + env: + ARTIFACT_DIR: '$(Pipeline.Workspace)/Staging' + OUT_DIR: '$(Pipeline.Workspace)/FinalPackage' + inputs: + scriptSource: 'inline' + script: | + import os + import shutil + from pathlib import Path + + dest_folder = os.environ['OUT_DIR'] + src_folder = os.environ['ARTIFACT_DIR'] + + if not os.path.exists(dest_folder): + os.makedirs(dest_folder) + + for content in Path(src_folder).iterdir(): + print(content) + shutil.copytree(content, dest_folder, dirs_exist_ok=True) + - task: CopyFiles@2 - displayName: Move Logs into Staging + displayName: Copy License inputs: - sourceFolder: '$(Pipeline.Workspace)/Logs' - contents: | - **/*.txt - targetFolder: '$(Pipeline.Workspace)/Staging' + sourceFolder: '$(Pipeline.Workspace)/Extras' + contents: 'License.txt' + targetFolder: '$(Pipeline.Workspace)/FinalPackage' flattenFolders: true - # Files in staging exists in this format - # {FLAVOR} - # -- CryptoPkg - # ---- {TARGET}_{TOOLCHAIN} - # ------ Crypto(Pei|Dxe|Smm).efi - # ------ {GUID} - # -------- DEBUG - # ---------- Crypto(Pei|Dxe|Smm).efi - # -------- OUTPUT - # ---------- Crypto(Pei|Dxe|Smm).efi - # ---------- Crypto(Pei|Dxe|Smm).depex - # -- UPDATE_LOG.txt - # -- CI_BUILDLOG.txt - # -- edk2-BaseCryptoDriver.config.json - - # {FLAVOR} = ALL, TINY_SHA, ... - # {TARGET} = DEBUG, RELEASE - # {TOOLCHAIN} = VS2019, VS2017 - # {ARCH} = IA32, X64, AARCH64 - - # We need them laid out like this - # {FLAVOR} - # -- {TARGET} - # ---- {ARCH} - # ------- Crypto(Pei|Dxe|Smm).efi - # ---- BuildReport.txt - # ---- Crypto(Pei|Dxe|Smm).depex - # License.txt - # Readme.md - - - task: PythonScript@0 - displayName: Assemble Release Package - inputs: - scriptSource: filePath - scriptPath: AssembleNugetPackage.py - arguments: > # Use this to avoid newline characters in multiline string - -v - -f - -l - -e $(Pipeline.Workspace)/Extras/License.txt - -o $(Pipeline.Workspace)/FinalPackage - $(Pipeline.Workspace)/Staging - - task: PublishPipelineArtifact@1 displayName: Publish Binaries inputs: diff --git a/.gitignore b/.gitignore index f15a92a0..3f2e0c3b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ Build/ Conf/ NugetPackage/ +Bundle/ diff --git a/AssembleNugetPackage.py b/AssembleNugetPackage.py deleted file mode 100644 index 6cf4b910..00000000 --- a/AssembleNugetPackage.py +++ /dev/null @@ -1,194 +0,0 @@ -# @file AssembleNugetPackage.py -# Script to rearrange a standard build output directory -# into the correct format for Nuget publication. -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## -import os -import glob -import argparse -import shutil -import logging - -from CommonBuildSettngs import CommonPlatform - -SCRIPT_PATH = os.path.abspath(__file__) -SCRIPT_DIR = os.path.dirname(SCRIPT_PATH) - - -def parse_args(): - arg_parse = argparse.ArgumentParser(f"{CommonPlatform.BaseName} AssembleNugetPackage.py", - description="assembles the final directory structure format for " - + f"the {CommonPlatform.BaseName} Nuget feed") - - def validate_abs_path(path_arg: str): - if not os.path.exists(path_arg): - raise ValueError(f"path '{path_arg}' is not valid") - return os.path.abspath(path_arg) - - arg_parse.add_argument(dest="input_dir", type=validate_abs_path, - help="the directory containing the build files to assemble") - default_output = os.path.join(SCRIPT_DIR, "NugetPackage") - arg_parse.add_argument("-o", "--output-dir", dest="output_dir", - default=default_output, - help=f"destination path to assemble the package. [default: '{default_output}']") - arg_parse.add_argument("-e", "--extra", dest="extra_paths", action="append", - type=validate_abs_path, default=[], - help="extra files/dirs to include in the root of the package") - arg_parse.add_argument("-f", "--force", dest="force", - default=False, action="store_true", - help="if set, will delete the output directory, if present") - arg_parse.add_argument("-l", "--logs", dest="copy_logs", - default=False, action="store_true", - help="if set, will also copy all of the generated log files into the package") - arg_parse.add_argument("-v", "--verbose", dest="verbose", - default=False, action="store_true", - help="if set, enables verbose logging") - - # TODO: Maybe support filtering SOME archs/flavors/targets/etc? - - args = arg_parse.parse_args() - return (args, arg_parse) - - -def main(): - (args, arg_parse) = parse_args() - if args.verbose: - logging.getLogger().setLevel(logging.DEBUG) - else: - logging.getLogger().setLevel(logging.INFO) - logging.debug(arg_parse.format_usage()) - logging.info(f"Assembling package at: {args.output_dir}...") - - # Create the output directory. - if os.path.exists(args.output_dir): - if args.force: - logging.info("Removing existing directory...") - shutil.rmtree(args.output_dir) - else: - raise ValueError(f"output path '{args.output_dir}' already exists") - os.makedirs(args.output_dir, exist_ok=True) - - # [Optional] Copy base logs - if args.copy_logs: - args.extra_paths += [ - os.path.join(args.input_dir, "SETUPLOG.txt"), - os.path.join(args.input_dir, "UPDATE_LOG.txt"), - ] - - # Copy all the files as they are found in the build directory. - for flavor in CommonPlatform.AvailableFlavors: - flavor_dir_name = f"{CommonPlatform.BaseName}_{flavor}" - flavor_input_dir = os.path.join(args.input_dir, flavor_dir_name) - - # May need to merge multiple Build output directories - if not os.path.exists(flavor_input_dir): - os.makedirs(flavor_input_dir) - for folder in glob.glob(f"{args.input_dir}/{CommonPlatform.BaseName}_*_{flavor}"): - logging.debug(f"MERGING FOLDER: '{folder}'") - for content in os.listdir(folder): - shutil.move(os.path.join(folder,content), flavor_input_dir) - - logging.debug(f"FLAVOR PATH: '{flavor_input_dir}'") - - for target in CommonPlatform.TargetsSupported: - target_dest_path = os.path.join(args.output_dir, flavor, target) - if not os.path.exists(target_dest_path): - os.makedirs(target_dest_path) - - # Copy all Depexes - # These are not unique per arch, so just copy the first one of each module - # we encounter. - search_path = os.path.join(flavor_input_dir, f"{target}*", "**") - logging.debug(f"TARGET SEARCH PATH: '{search_path}'") - for path in glob.iglob(os.path.join(search_path, "Crypto*.depex"), recursive=True): - output_path = os.path.join(target_dest_path, os.path.basename(path)) - if not os.path.exists(output_path): - logging.debug(f"{path} -> {output_path}") - shutil.copy(path, output_path) - - # Copy all driver binaries - for arch in CommonPlatform.ArchSupported: - arch_dest_path = os.path.join(target_dest_path, arch) - search_path = os.path.join(flavor_dir_name, f"{target}*", arch) - logging.debug(f"ARCH SEARCH PATH: '{search_path}'") - for path in glob.iglob(os.path.join(args.input_dir, search_path, "Crypto*.efi")): - if not os.path.exists(arch_dest_path): - os.makedirs(arch_dest_path) - logging.debug(f"{path} -> {arch_dest_path}") - shutil.copy(path, arch_dest_path) - # Get Map and PDB files - for map_path in glob.iglob(os.path.join(args.input_dir, search_path, "CryptoBinPkg", "Driver", "*", "Output", "Crypto*.map")): - pdb_path = map_path[:-3] + 'pdb' - shutil.copy(map_path, arch_dest_path) - if os.path.exists(pdb_path): - shutil.copy(pdb_path, arch_dest_path) - - # [Optional] Copy logs - if args.copy_logs: - log_file = os.path.join(args.input_dir, - f"BUILDLOG_{CommonPlatform.BaseName}_{flavor}_{target}.txt") - if os.path.isfile(log_file): - logging.debug(f"{log_file} -> {target_dest_path}") - shutil.copy(log_file, target_dest_path) - - search_path = os.path.join(flavor_input_dir, f"{target}*") - for path in glob.iglob(os.path.join(search_path, "BUILD_REPORT.TXT"), recursive=True): - output_path = os.path.join(target_dest_path, os.path.basename(path)) - if not os.path.exists(output_path): - logging.debug(f"{path} -> {output_path}") - shutil.copy(path, output_path) - - # Add additional outgenerated files for the crypto binary - # Find the Driver folder - autogen_dir_name = f"CryptoBinPkg/Driver/" - autogen_input_dir = os.getcwd() - autogen_input_dir = os.path.join(autogen_input_dir, autogen_dir_name) - logging.debug(f"AUTOGEN PATH: '{autogen_input_dir}'") - - # Create the Driver folder - autogen_dest_path = os.path.join(args.output_dir, "Driver") - if not os.path.exists(autogen_dest_path): - os.makedirs(autogen_dest_path) - - # Copy the files in the Driver directory - for file in os.listdir(autogen_input_dir): - fullpath = os.path.join(autogen_input_dir, file) - # Skip directories for now - if os.path.isdir(fullpath): - continue - filename = os.fsdecode(file) - input_path = os.path.join(autogen_input_dir, filename) - output_path = os.path.join(autogen_dest_path, filename) - logging.debug(f"OUTPUT PATH: {output_path}") - shutil.copy(input_path, output_path) - - # Create the Driver/Bin folder - autogen_input_dir = os.path.join(autogen_input_dir, "Bin") - autogen_dest_path = os.path.join(autogen_dest_path, "Bin") - if not os.path.exists(autogen_dest_path): - os.makedirs(autogen_dest_path) - - # Copy all files in the Bin folder - for file in os.listdir(autogen_input_dir): - filename = os.fsdecode(file) - # Skip the temporary files - if filename[0:4] == "temp": - continue - input_path = os.path.join(autogen_input_dir, filename) - output_path = os.path.join(autogen_dest_path, filename) - logging.debug(f"OUTPUT PATH: {output_path}") - shutil.copy(input_path, output_path) - - # Finally, copy any extra paths to the output. - for extra_path in args.extra_paths: - if os.path.isfile(extra_path): - logging.debug(f"{extra_path} -> {args.output_dir}") - shutil.copy(extra_path, args.output_dir) - else: - logging.warning(f"Extra path could not be found: {extra_path}") - - -if __name__ == "__main__": - main() diff --git a/CommonBuildSettngs.py b/CommonBuildSettings.py similarity index 100% rename from CommonBuildSettngs.py rename to CommonBuildSettings.py diff --git a/CryptoBinPkg/CryptoBinPkg.dsc b/CryptoBinPkg/CryptoBinPkg.dsc index e0debbaa..e7e3cd80 100644 --- a/CryptoBinPkg/CryptoBinPkg.dsc +++ b/CryptoBinPkg/CryptoBinPkg.dsc @@ -244,6 +244,10 @@ GCC:*_*_*_CC_FLAGS = -D ENABLE_MD5_DEPRECATED_INTERFACES RVCT:*_*_*_CC_FLAGS = -DENABLE_MD5_DEPRECATED_INTERFACES !endif +MSFT:RELEASE_*_*_CC_FLAGS = /Zi +MSFT:RELEASE_*_*_DLINK_FLAGS = /DEBUG /OPT:REF /OPT:ICF=10 /ALT /PDBALTPATH:$(MODULE_NAME) +MSFT:RELEASE_*_*_ASM_FLAGS = /Zi +MSFT:RELEASE_*_*_NASM_FLAGS = -g [BuildOptions.X64.EDKII.PEIM, BuildOptions.AARCH64.EDKII.PEIM] MSFT:*_*_*_DLINK_FLAGS = /FILEALIGN:0x1000 # meet requirement for PEIM section and file alignment to match. diff --git a/CryptoBinPkg/Plugin/BundleCrypto/BundleCrypto.py b/CryptoBinPkg/Plugin/BundleCrypto/BundleCrypto.py new file mode 100644 index 00000000..9e986033 --- /dev/null +++ b/CryptoBinPkg/Plugin/BundleCrypto/BundleCrypto.py @@ -0,0 +1,77 @@ +## @file BundleCrypto.py +# Plugin to Bundle the CryptoBin build output +# +## +# Copyright (c) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## +from edk2toolext.environment.plugintypes.uefi_build_plugin import IUefiBuildPlugin +from pathlib import Path + +class BundleCrypto(IUefiBuildPlugin): + + def do_post_build(self, thebuilder): + #Path to Build output + build_path = Path(thebuilder.env.GetValue("BUILD_OUTPUT_BASE")) + #Path to where each bundle will be placed + bundle_dir = Path(thebuilder.ws) / "Bundle" / thebuilder.flavor / thebuilder.env.GetValue("TARGET") + + # Copy .pdb, .depex, .map, .efi files for each architecture built. + arch_list = thebuilder.env.GetValue("TARGET_ARCH") + for arch in arch_list.split(" "): + target_dir = bundle_dir / arch + target_dir.mkdir(parents=True, exist_ok=True) + arch_build_path = build_path / arch + files = list(arch_build_path.rglob("*.pdb")) \ + + list(arch_build_path.rglob("*.map")) \ + + list(arch_build_path.rglob("*.efi")) \ + + list(arch_build_path.rglob("*.depex")) + for file in files: + # pdb exists in DEBUG and OUTPUT directory. Same file. + file_out = target_dir / file.name + if file.parent.name != "OUTPUT": + continue + # If it exists and has the same file identifier, skip it. + if file_out.exists() and file.stat().st_ino == file_out.stat().st_ino: + continue + if "vc1" in file.name.lower(): + continue + + file_out.unlink(missing_ok=True) + file_out.hardlink_to(file) + + # Copy the Build report + br_src = Path(thebuilder.env.GetValue("BUILD_OUTPUT_BASE"), "BUILD_REPORT.TXT") + br_dst = bundle_dir / f"BUILD_REPORT_{arch_list.replace(' ', '_')}.TXT" + if not br_dst.exists() or br_dst.stat().st_ino != br_src.stat().st_ino: + br_dst.unlink(missing_ok=True) + br_dst.hardlink_to(br_src) + + # Copy the build log + bl_src = Path(thebuilder.ws) / "Build" / f"BUILDLOG_CryptoBin_{thebuilder.flavor}_{thebuilder.env.GetValue('TARGET')}.txt" + bl_dst = bundle_dir / f"BUILDLOG_CryptoBin_{thebuilder.flavor}_{thebuilder.env.GetValue('TARGET')}_{arch_list.replace(' ', '_')}.txt" + if not bl_dst.exists() or bl_dst.stat().st_ino != bl_src.stat().st_ino: + bl_dst.unlink(missing_ok=True) + bl_dst.hardlink_to(bl_src) + + # Copy CryptoBinPkg/Driver/* contents (excluding Packaging) + driver_src = Path(thebuilder.edk2path.GetAbsolutePathOnThisSystemFromEdk2RelativePath("CryptoBinPkg")) / "Driver" + driver_out = Path(thebuilder.ws) / "Bundle" / "Driver" + driver_out.mkdir(parents=True, exist_ok=True) + for file in driver_src.iterdir(): + if file.is_dir(): + continue + file_out = driver_out / file.name + file_out.unlink(missing_ok=True) + file_out.hardlink_to(file) + driver_bin_src = driver_src / "Bin" + driver_bin_out = driver_out / "Bin" + driver_bin_out.mkdir(parents=True, exist_ok=True) + for file in driver_bin_src.iterdir(): + if file.is_dir(): + continue + file_out = driver_bin_out / file.name + file_out.unlink(missing_ok=True) + file_out.hardlink_to(file) + + return 0 diff --git a/CryptoBinPkg/Plugin/BundleCrypto/BundleCrypto_plug_in.json b/CryptoBinPkg/Plugin/BundleCrypto/BundleCrypto_plug_in.json new file mode 100644 index 00000000..64ea1738 --- /dev/null +++ b/CryptoBinPkg/Plugin/BundleCrypto/BundleCrypto_plug_in.json @@ -0,0 +1,5 @@ +{ + "scope": "crypto-bundle", + "name": "Bundle Crypto Contents", + "module": "BundleCrypto" +} diff --git a/MultiFlavorBuild.py b/MultiFlavorBuild.py index 3bf47ae0..0e20e508 100644 --- a/MultiFlavorBuild.py +++ b/MultiFlavorBuild.py @@ -13,7 +13,7 @@ from edk2toolext.invocables.edk2_platform_build import BuildSettingsManager from edk2toollib.utility_functions import RunPythonScript -from CommonBuildSettngs import CommonPlatform, CommonSettingsManager +from CommonBuildSettings import CommonPlatform, CommonSettingsManager # ####################################################################################### # @@ -58,12 +58,16 @@ def validate_flavors(flavor_arg: str): parserObj.add_argument("-f", "--flavor", dest="flavor", type=validate_flavors, default=CommonPlatform.AvailableFlavors, help="flavor(s) for the build {%s}" % ",".join(CommonPlatform.AvailableFlavors)) + parserObj.add_argument("-b", "--bundle", dest="bundle", action="store_true", + default=False, + help="Bundles the build output into the directory structure for the Crypto binary distribution.") def RetrieveCommandLineOptions(self, args): self.arch = args.arch self.target = args.target self.flavor = args.flavor self.stop = args.stop + self.bundle = args.bundle def GetWorkspaceRoot(self): ''' get WorkspacePath ''' @@ -110,6 +114,8 @@ def Go(self, WorkSpace, PackagesPath, PInHelper, PInManager): params += [f"TOOL_CHAIN_TAG={toolchain}"] params += ["-t", target] params += ["-a", ",".join(arches)] + if self.bundle: + params += ["-b"] current_build = f"{flavor} {target}" logging.log(edk2_logging.SECTION, f"Building {current_build}") @@ -128,6 +134,8 @@ def Go(self, WorkSpace, PackagesPath, PInHelper, PInManager): params += [f"TOOL_CHAIN_TAG=GCC5"] params += ["-t", target] params += ["-a", "AARCH64"] + if self.bundle: + params += ["-b"] current_build = f"{flavor} {target}" logging.log(edk2_logging.SECTION, f"Building {current_build}") diff --git a/Readme.rst b/Readme.rst index 9fa8ebf6..4ad2611f 100644 --- a/Readme.rst +++ b/Readme.rst @@ -81,10 +81,9 @@ The steps are: 2) Run the ``MultiFlavorBuild.py`` script with ``--setup`` 3) Run the ``MultiFlavorBuild.py`` script with ``--update`` 4) Run the ``MultiFlavorBuild.py`` script with whatever flavors and architectures you would like in - your binary pacakge. This is the primary build and may take a while -5) Run the ``AssembleNugetPackage.py`` script and point at the ``Build`` directory for your local - build (completed in step 4). There are extra parameters you can use to configure the package, - documented in the help for the script + your binary package. This is the primary build and may take a while +5) Run the ``MultiFlavorBuild.py`` script with ``--bundle`` to create the Nuget package layout + in the ``Bundle`` directory Releasing a Pipeline Build diff --git a/SingleFlavorBuild.py b/SingleFlavorBuild.py index 4facb11a..ec614c18 100644 --- a/SingleFlavorBuild.py +++ b/SingleFlavorBuild.py @@ -10,7 +10,7 @@ from edk2toolext.environment.uefi_build import UefiBuilder from edk2toolext.invocables.edk2_platform_build import BuildSettingsManager -from CommonBuildSettngs import CommonPlatform, CommonSettingsManager +from CommonBuildSettings import CommonPlatform, CommonSettingsManager # ####################################################################################### # @@ -39,11 +39,16 @@ def AddCommandLineOptions(self, parserObj): parserObj.add_argument(dest="flavor", type=str, choices=CommonPlatform.AvailableFlavors, help="the flavor to build for the Crypto binary distribution") + parserObj.add_argument("-b", "--bundle", dest="bundle", action="store_true", + default=False, + help="Bundles the build output into the directory structure for the Crypto binary distribution.") + def RetrieveCommandLineOptions(self, args): self.flavor = args.flavor self.target = args.target self.arch = args.arch + self.bundle = args.bundle def GetWorkspaceRoot(self): ''' get WorkspacePath ''' @@ -55,6 +60,8 @@ def GetPackagesPath(self): def GetActiveScopes(self): ''' return tuple containing scopes that should be active for this process ''' + if self.bundle: + return CommonPlatform.Scopes + ("crypto-bundle",) return CommonPlatform.Scopes def GetBaseName(self):