From 74bf7f55c1e95574334043fad5cc341afcf4f377 Mon Sep 17 00:00:00 2001 From: Kun Qin Date: Wed, 24 Jul 2024 13:13:02 -0700 Subject: [PATCH] BaseTools: Adding cross compilation of BaseTool for Windows ARM/ARM64 This change adds the support of crossbuilding basetool for Windows ARM/ ARM64 systems, which will enable the generally available pipeline agents to build binary tools and make releases as they see fit. The EDK2 base tools build script is also updated to support cross compilation using this script. The crossbuilt binary output is tested on Windows ARM based hardware systems. Cc: Rebecca Cran Cc: Liming Gao Cc: Bob Feng Cc: Yuwei Chen Signed-off-by: Kun Qin --- BaseTools/.gitignore | 1 + BaseTools/Edk2ToolsBuild.py | 116 ++++++++++++++++-- BaseTools/Source/C/Makefile | 2 + .../C/VfrCompile/Pccts/antlr/AntlrMS.mak | 15 ++- .../Source/C/VfrCompile/Pccts/dlg/DlgMS.mak | 15 ++- 5 files changed, 135 insertions(+), 14 deletions(-) diff --git a/BaseTools/.gitignore b/BaseTools/.gitignore index ec9c69f8b25f..018612825038 100644 --- a/BaseTools/.gitignore +++ b/BaseTools/.gitignore @@ -17,5 +17,6 @@ Source/C/VfrCompile/VfrTokens.h Source/C/bin/ Source/C/libs/ Bin/Win32 +Bin/Win64 Lib BaseToolsBuild/ \ No newline at end of file diff --git a/BaseTools/Edk2ToolsBuild.py b/BaseTools/Edk2ToolsBuild.py index 2a8a53b0d649..d511b1a2a342 100644 --- a/BaseTools/Edk2ToolsBuild.py +++ b/BaseTools/Edk2ToolsBuild.py @@ -12,10 +12,11 @@ import logging import argparse import multiprocessing +import shutil from edk2toolext import edk2_logging -from edk2toolext.environment import self_describing_environment +from edk2toolext.environment import self_describing_environment, shell_environment from edk2toolext.base_abstract_invocable import BaseAbstractInvocable -from edk2toollib.utility_functions import RunCmd +from edk2toollib.utility_functions import RunCmd, GetHostInfo from edk2toollib.windows.locate_tools import QueryVcVariables @@ -26,8 +27,12 @@ def ParseCommandLineOptions(self): ParserObj = argparse.ArgumentParser() ParserObj.add_argument("-t", "--tool_chain_tag", dest="tct", default="VS2022", help="Set the toolchain used to compile the build tools") + ParserObj.add_argument("-a", "--target_arch", dest="arch", default=None, choices=[None, 'IA32', 'X64', 'ARM', 'AARCH64'], + help="Specify the architecture of the built base tools. Not specifying this will fall back to the default " + "behavior, for Windows builds, IA32 target will be built, for Linux builds, target arch will be the same as host arch.") args = ParserObj.parse_args() self.tool_chain_tag = args.tct + self.target_arch = args.arch def GetWorkspaceRoot(self): ''' Return the workspace root for initializing the SDE ''' @@ -110,6 +115,103 @@ def Go(self): shell_env.set_shell_var("PYTHON_COMMAND", pc) if self.tool_chain_tag.lower().startswith("vs"): + if self.target_arch is None: + # Put a default as IA32 + self.target_arch = "IA32" + + if self.target_arch == "IA32": + VcToolChainArch = "x86" + TargetInfoArch = "x86" + OutputDir = "Win32" + elif self.target_arch == "ARM": + VcToolChainArch = "x86_arm" + TargetInfoArch = "ARM" + OutputDir = "Win32" + elif self.target_arch == "X64": + VcToolChainArch = "amd64" + TargetInfoArch = "x86" + OutputDir = "Win64" + elif self.target_arch == "AARCH64": + VcToolChainArch = "amd64_arm64" + TargetInfoArch = "ARM" + OutputDir = "Win64" + else: + raise NotImplementedError() + + self.OutputDir = os.path.join( + shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", OutputDir) + + # compiled tools need to be added to path because antlr is referenced + HostInfo = GetHostInfo() + if TargetInfoArch == HostInfo.arch: + # not cross compiling + shell_env.insert_path(self.OutputDir) + else: + # cross compiling: + # as the VfrCompile tool is needed in the build process, we need + # to build one for the host system, then add the path to the + # tools to the PATH environment variable + shell_environment.CheckpointBuildVars() + if HostInfo.arch == "x86" and HostInfo.bit == "64": + host_arch = "X64" + host_toolchain_arch = "amd64" + TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64") + elif HostInfo.arch == "x86" and HostInfo.bit == "32": + host_arch = "IA32" + host_toolchain_arch = "x86" + TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32") + elif HostInfo.arch == "ARM" and HostInfo.bit == "64": + host_arch = "AARCH64" + host_toolchain_arch = "amd64_arm64" + TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64") + elif HostInfo.arch == "ARM" and HostInfo.bit == "32": + host_arch = "ARM" + host_toolchain_arch = "x86_arm" + TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32") + else: + raise Exception("Unsupported host system. %s %s" % (HostInfo.arch, HostInfo.bit)) + + interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"] + interesting_keys.extend( + ["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"]) + interesting_keys.extend( + ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"]) + vc_vars = QueryVcVariables( + interesting_keys, host_toolchain_arch, vs_version=self.tool_chain_tag.lower()) + for key in vc_vars.keys(): + logging.debug(f"Var - {key} = {vc_vars[key]}") + if key.lower() == 'path': + shell_env.set_path(vc_vars[key]) + else: + shell_env.set_shell_var(key, vc_vars[key]) + + # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where + # this script is BUILDING the base tools. + shell_env.set_shell_var('HOST_ARCH', host_arch) + shell_env.insert_path(TempOutputDir) + + # All set, build the tools for the host system. + ret = RunCmd('nmake.exe', None, + workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH")) + if ret != 0: + raise Exception("Failed to build base tools for host system.") + + # Copy the output to a temp directory + TempFolder = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "BaseToolsBuild", "Temp") + if not os.path.exists(TempFolder): + os.makedirs(TempFolder) + for file in os.listdir(TempOutputDir): + shutil.copy(os.path.join(TempOutputDir, file), TempFolder) + + # Clean up the build output + ret = RunCmd('nmake.exe', 'cleanall', + workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH")) + + # Remove the entire TempOutputDir + shutil.rmtree(TempOutputDir) + + shell_environment.RevertBuildVars() + shell_env.insert_path(TempFolder) # # Update environment with required VC vars. interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"] @@ -118,7 +220,7 @@ def Go(self): interesting_keys.extend( ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"]) vc_vars = QueryVcVariables( - interesting_keys, 'x86', vs_version=self.tool_chain_tag.lower()) + interesting_keys, VcToolChainArch, vs_version=self.tool_chain_tag.lower()) for key in vc_vars.keys(): logging.debug(f"Var - {key} = {vc_vars[key]}") if key.lower() == 'path': @@ -126,11 +228,9 @@ def Go(self): else: shell_env.set_shell_var(key, vc_vars[key]) - self.OutputDir = os.path.join( - shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32") - - # compiled tools need to be added to path because antlr is referenced - shell_env.insert_path(self.OutputDir) + # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where + # this script is BUILDING the base tools. + shell_env.set_shell_var('HOST_ARCH', self.target_arch) # Actually build the tools. output_stream = edk2_logging.create_output_stream() diff --git a/BaseTools/Source/C/Makefile b/BaseTools/Source/C/Makefile index a376d32e220e..0f26de0aa19a 100644 --- a/BaseTools/Source/C/Makefile +++ b/BaseTools/Source/C/Makefile @@ -4,7 +4,9 @@ # Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent # +!IFNDEF HOST_ARCH HOST_ARCH = IA32 +!ENDIF !INCLUDE Makefiles\ms.common diff --git a/BaseTools/Source/C/VfrCompile/Pccts/antlr/AntlrMS.mak b/BaseTools/Source/C/VfrCompile/Pccts/antlr/AntlrMS.mak index 6fc4d5c15d6c..26c590adf4b0 100644 --- a/BaseTools/Source/C/VfrCompile/Pccts/antlr/AntlrMS.mak +++ b/BaseTools/Source/C/VfrCompile/Pccts/antlr/AntlrMS.mak @@ -8,6 +8,15 @@ PCCTS_HOME=$(BASE_TOOLS_PATH)\Source\C\VfrCompile\Pccts ANTLR_SRC=$(PCCTS_HOME)\antlr PCCTS_H=$(PCCTS_HOME)\h +!IFNDEF HOST_ARCH +HOST_ARCH = IA32 +!ENDIF + +!IF "$(HOST_ARCH)"=="IA32" || "$(HOST_ARCH)"=="ARM" +SYS_BIN_PATH=$(EDK_TOOLS_PATH)\Bin\Win32 +!ELSE +SYS_BIN_PATH=$(EDK_TOOLS_PATH)\Bin\Win64 +!ENDIF # Support directories SET=$(PCCTS_HOME)\support\set @@ -27,10 +36,10 @@ SUPPORT_OBJS = set.obj # Dependencies -$(EDK_TOOLS_PATH)\Bin\Win32\antlr.exe: $(ANTLR_OBJS) $(SUPPORT_OBJS) +$(SYS_BIN_PATH)\antlr.exe: $(ANTLR_OBJS) $(SUPPORT_OBJS) $(CC) $(CFLAGS) -Feantlr.exe $(ANTLR_OBJS) $(SUPPORT_OBJS) - -@if not exist $(EDK_TOOLS_PATH)\Bin\Win32 mkdir $(EDK_TOOLS_PATH)\Bin\Win32 - copy antlr.exe $(EDK_TOOLS_PATH)\Bin\Win32 + -@if not exist $(SYS_BIN_PATH) mkdir $(SYS_BIN_PATH) + copy antlr.exe $(SYS_BIN_PATH) antlr.obj: $(ANTLR_SRC)\antlr.c \ diff --git a/BaseTools/Source/C/VfrCompile/Pccts/dlg/DlgMS.mak b/BaseTools/Source/C/VfrCompile/Pccts/dlg/DlgMS.mak index c2cac00f53df..f2a31a775e2a 100644 --- a/BaseTools/Source/C/VfrCompile/Pccts/dlg/DlgMS.mak +++ b/BaseTools/Source/C/VfrCompile/Pccts/dlg/DlgMS.mak @@ -12,6 +12,15 @@ PCCTS_H=$(PCCTS_HOME)\h # Support directories SET=$(PCCTS_HOME)\support\set +!IFNDEF HOST_ARCH +HOST_ARCH = IA32 +!ENDIF + +!IF "$(HOST_ARCH)"=="IA32" || "$(HOST_ARCH)"=="ARM" +SYS_BIN_PATH=$(EDK_TOOLS_PATH)\Bin\Win32 +!ELSE +SYS_BIN_PATH=$(EDK_TOOLS_PATH)\Bin\Win64 +!ENDIF # Compiler stuff CC = cl @@ -26,10 +35,10 @@ SUPPORT_OBJS = set.obj # Dependencies -$(EDK_TOOLS_PATH)\Bin\Win32\dlg.exe: $(DLG_OBJS) $(SUPPORT_OBJS) +$(SYS_BIN_PATH)\dlg.exe: $(DLG_OBJS) $(SUPPORT_OBJS) $(CC) $(CFLAGS) -Fedlg.exe $(DLG_OBJS) $(SUPPORT_OBJS) - -@if not exist $(EDK_TOOLS_PATH)\Bin\Win32 mkdir $(EDK_TOOLS_PATH)\Bin\Win32 - copy dlg.exe $(EDK_TOOLS_PATH)\Bin\Win32 + -@if not exist $(SYS_BIN_PATH) mkdir $(SYS_BIN_PATH) + copy dlg.exe $(SYS_BIN_PATH) dlg_p.obj: $(DLG_SRC)\dlg_p.c \ $(PCCTS_H)\antlr.h \