From 3e45f9900f05bc74a57da3c4a845015c1c2ded2f Mon Sep 17 00:00:00 2001 From: Bret Barkelew Date: Mon, 22 Mar 2021 13:42:47 -0700 Subject: [PATCH] Sync pipelines, requirements, and documents with Basecore --- .azurepipelines/Ubuntu-GCC5.yml | 4 ++ .azurepipelines/Ubuntu-PatchCheck.yml | 4 +- .../templates/platform-build-run-steps.yml | 8 ++- .../templates/pr-gate-build-job.yml | 4 +- .azurepipelines/templates/pr-gate-steps.yml | 18 ++---- .../templates/spell-check-prereq-steps.yml | 2 +- .gitignore | 6 +- .pytool/CISettings.py | 57 ++++++++++++++----- pip-requirements.txt | 6 +- 9 files changed, 73 insertions(+), 36 deletions(-) diff --git a/.azurepipelines/Ubuntu-GCC5.yml b/.azurepipelines/Ubuntu-GCC5.yml index d51bd0091b..32ec3003b0 100644 --- a/.azurepipelines/Ubuntu-GCC5.yml +++ b/.azurepipelines/Ubuntu-GCC5.yml @@ -30,4 +30,8 @@ jobs: tool_chain_tag: 'GCC5' vm_image: 'ubuntu-latest' arch_list: "IA32,X64,ARM,AARCH64" + # MU_CHANGE + extra_steps: + - script: sudo apt-get install mingw-w64 + displayName: Install Windows Resource Compiler for Linux diff --git a/.azurepipelines/Ubuntu-PatchCheck.yml b/.azurepipelines/Ubuntu-PatchCheck.yml index 995039fadb..0e6c112e33 100644 --- a/.azurepipelines/Ubuntu-PatchCheck.yml +++ b/.azurepipelines/Ubuntu-PatchCheck.yml @@ -5,7 +5,7 @@ # NOTE: This example monitors pull requests against the edk2-ci branch. Most # environments would replace 'edk2-ci' with 'master'. # -# Copyright (c) 2019, Intel Corporation. All rights reserved.
+# Copyright (c) 2019 - 2020, Intel Corporation. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent # # https://github.com/tianocore @@ -29,7 +29,7 @@ steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.7.x' + versionSpec: '3.9.x' # MU_CHANGE architecture: 'x64' - script: | diff --git a/.azurepipelines/templates/platform-build-run-steps.yml b/.azurepipelines/templates/platform-build-run-steps.yml index 447ac5e1b6..0fe0306899 100644 --- a/.azurepipelines/templates/platform-build-run-steps.yml +++ b/.azurepipelines/templates/platform-build-run-steps.yml @@ -42,7 +42,7 @@ steps: - task: UsePythonVersion@0 inputs: - versionSpec: "3.8.x" + versionSpec: "3.9.x" # MU_CHANGE architecture: "x64" - script: pip install -r pip-requirements.txt --upgrade @@ -51,6 +51,12 @@ steps: # Set default - bash: echo "##vso[task.setvariable variable=pkg_count]${{ 1 }}" +# Fetch the target branch so that pr_eval can diff them. +# Seems like azure pipelines/github changed checkout process in nov 2020. +- script: git fetch origin $(System.PullRequest.targetBranch) + displayName: fetch target branch + condition: eq(variables['Build.Reason'], 'PullRequest') + # trim the package list if this is a PR - task: CmdLine@1 displayName: Check if ${{ parameters.build_pkg }} need testing diff --git a/.azurepipelines/templates/pr-gate-build-job.yml b/.azurepipelines/templates/pr-gate-build-job.yml index 86a3dba014..5a873fe0c8 100644 --- a/.azurepipelines/templates/pr-gate-build-job.yml +++ b/.azurepipelines/templates/pr-gate-build-job.yml @@ -4,7 +4,7 @@ # template file used to build supported packages. # # Copyright (c) Microsoft Corporation. -# Copyright (c) 2020, ARM Limited. All rights reserved.
+# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent ## @@ -12,6 +12,7 @@ parameters: tool_chain_tag: '' vm_image: '' arch_list: '' + extra_steps: [] # MU_CHANGE # Build step jobs: @@ -40,6 +41,7 @@ jobs: vmImage: ${{ parameters.vm_image }} steps: + - ${{ parameters.extra_steps }} # MU_CHANGE - template: pr-gate-steps.yml parameters: tool_chain_tag: ${{ parameters.tool_chain_tag }} diff --git a/.azurepipelines/templates/pr-gate-steps.yml b/.azurepipelines/templates/pr-gate-steps.yml index 82424d57e4..6a945b7ace 100644 --- a/.azurepipelines/templates/pr-gate-steps.yml +++ b/.azurepipelines/templates/pr-gate-steps.yml @@ -20,7 +20,7 @@ steps: - task: UsePythonVersion@0 inputs: - versionSpec: '3.8.x' + versionSpec: '3.9.x' # MU_CHANGE architecture: 'x64' - script: pip install -r pip-requirements.txt --upgrade @@ -31,24 +31,18 @@ steps: echo "##vso[task.setvariable variable=pkgs_to_build]${{ parameters.build_pkgs }}" echo "##vso[task.setvariable variable=pkg_count]${{ 1 }}" -# MU_CHANGE [BEGIN] - Workaround an azure pipelines bug. -# $System.PullRequest.targetBranch looks like "dev/whatever" on GitHub, -# but looks like "refs/heads/dev/whatever" on DevOps. The DevOps version -# can't be used for comparison for the PR Eval. -- powershell: - $TargetBranch = "$(System.PullRequest.targetBranch)".replace('refs/heads/', ''); - Write-Host "##vso[task.setvariable variable=pr_compare_branch]origin/$TargetBranch"; - displayName: Workaround for Branch Names +# Fetch the target branch so that pr_eval can diff them. +# Seems like azure pipelines/github changed checkout process in nov 2020. +- script: git fetch origin $(System.PullRequest.targetBranch) + displayName: fetch target branch condition: eq(variables['Build.Reason'], 'PullRequest') -# MU_CHANGE [END] # trim the package list if this is a PR - task: CmdLine@1 displayName: Check if ${{ parameters.build_pkgs }} need testing inputs: filename: stuart_pr_eval - # MU_CHANGE - Workaround an azure pipelines bug. - arguments: -c .pytool/CISettings.py -p ${{ parameters.build_pkgs }} --pr-target $(pr_compare_branch) --output-csv-format-string "##vso[task.setvariable variable=pkgs_to_build;isOutpout=true]{pkgcsv}" --output-count-format-string "##vso[task.setvariable variable=pkg_count;isOutpout=true]{pkgcount}" + arguments: -c .pytool/CISettings.py -p ${{ parameters.build_pkgs }} --pr-target origin/$(System.PullRequest.targetBranch) --output-csv-format-string "##vso[task.setvariable variable=pkgs_to_build;isOutpout=true]{pkgcsv}" --output-count-format-string "##vso[task.setvariable variable=pkg_count;isOutpout=true]{pkgcount}" condition: eq(variables['Build.Reason'], 'PullRequest') # install spell check prereqs diff --git a/.azurepipelines/templates/spell-check-prereq-steps.yml b/.azurepipelines/templates/spell-check-prereq-steps.yml index db651a9260..5805be4623 100644 --- a/.azurepipelines/templates/spell-check-prereq-steps.yml +++ b/.azurepipelines/templates/spell-check-prereq-steps.yml @@ -13,7 +13,7 @@ parameters: steps: - task: NodeTool@0 inputs: - versionSpec: '12.x' # mu_change [TCBZ2618] + versionSpec: '14.x' # MU_CHANGE #checkLatest: false # Optional condition: and(gt(variables.pkg_count, 0), succeeded()) diff --git a/.gitignore b/.gitignore index ea3eecc56d..edd8e6b620 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,11 @@ tags/ *.bak BuildConfig.conf +# Repo-specific +_TEMP_*/ +/Conf/ + # Ignore cloned dependencies /Common /Silicon /MU_BASECORE - -_TEMP_*/ diff --git a/.pytool/CISettings.py b/.pytool/CISettings.py index 0fae2d91f1..599b5bd760 100644 --- a/.pytool/CISettings.py +++ b/.pytool/CISettings.py @@ -21,16 +21,24 @@ def __init__(self): self.ActualTargets = [] self.ActualArchitectures = [] self.ActualToolChainTag = "" + self.UseBuiltInBaseTools = None + self.ActualScopes = None # ####################################################################################### # # Extra CmdLine configuration # # ####################################################################################### # def AddCommandLineOptions(self, parserObj): - pass + group = parserObj.add_mutually_exclusive_group() + group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools") + group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools") def RetrieveCommandLineOptions(self, args): - pass + super().RetrieveCommandLineOptions(args) + if args.force_piptools: + self.UseBuiltInBaseTools = True + if args.no_piptools: + self.UseBuiltInBaseTools = False # ####################################################################################### # # Default Support for this Ci Build # @@ -120,19 +128,38 @@ def SetTargets(self, list_of_requested_target): def GetActiveScopes(self): ''' return tuple containing scopes that should be active for this process ''' - scopes = ("cibuild", "edk2-build", "host-based-test") - - self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "") - - if GetHostInfo().os.upper() == "LINUX" and self.ActualToolChainTag.upper().startswith("GCC"): - if "AARCH64" in self.ActualArchitectures: - scopes += ("gcc_aarch64_linux",) - if "ARM" in self.ActualArchitectures: - scopes += ("gcc_arm_linux",) - if "RISCV64" in self.ActualArchitectures: - scopes += ("gcc_riscv64_unknown",) - - return scopes + if self.ActualScopes is None: + scopes = ("cibuild", "edk2-build", "host-based-test") + + self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "") + + is_linux = GetHostInfo().os.upper() == "LINUX" + + if self.UseBuiltInBaseTools is None: + is_linux = GetHostInfo().os.upper() == "LINUX" + # try and import the pip module for basetools + try: + import edk2basetools + self.UseBuiltInBaseTools = True + except ImportError: + self.UseBuiltInBaseTools = False + pass + + if self.UseBuiltInBaseTools == True: + scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',) + logging.warning("Using Pip Tools based BaseTools") + else: + logging.warning("Falling back to using in-tree BaseTools") + + if is_linux and self.ActualToolChainTag.upper().startswith("GCC"): + if "AARCH64" in self.ActualArchitectures: + scopes += ("gcc_aarch64_linux",) + if "ARM" in self.ActualArchitectures: + scopes += ("gcc_arm_linux",) + if "RISCV64" in self.ActualArchitectures: + scopes += ("gcc_riscv64_unknown",) + self.ActualScopes = scopes + return self.ActualScopes def GetRequiredSubmodules(self): ''' return iterable containing RequiredSubmodule objects. diff --git a/pip-requirements.txt b/pip-requirements.txt index d2d4b6c6e8..a24ef87cdd 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -12,5 +12,7 @@ # https://www.python.org/dev/peps/pep-0440/#version-specifiers ## -edk2-pytool-library~=0.10.8 # MU_CHANGE - update to 0.10.8 or later -edk2-pytool-extensions~=0.13.3 +edk2-pytool-library~=0.10.13 # MU_CHANGE - update to 0.10.13 or later +edk2-pytool-extensions~=0.14.0 # MU_CHANGE - update to 0.14.0 or later +edk2-basetools==0.1.2 +antlr4-python3-runtime==4.7.1