Sync pipelines, requirements, and documents with Basecore

.azurepipelines/Ubuntu-GCC5.yml

@@ -30,4 +30,8 @@ jobs:
     tool_chain_tag: 'GCC5'
     vm_image: 'ubuntu-latest'
     arch_list: "IA32,X64,ARM,AARCH64"
+    # MU_CHANGE
+    extra_steps:
+    - script: sudo apt-get install mingw-w64
+      displayName: Install Windows Resource Compiler for Linux
 

.azurepipelines/Ubuntu-PatchCheck.yml

@@ -5,7 +5,7 @@
 # NOTE: This example monitors pull requests against the edk2-ci branch.  Most
 # environments would replace 'edk2-ci' with 'master'.
 #
-# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2019 - 2020, Intel Corporation. All rights reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 #
 # https://github.com/tianocore
@@ -29,7 +29,7 @@ steps:
 
 - task: UsePythonVersion@0
   inputs:
-    versionSpec: '3.7.x'
+    versionSpec: '3.9.x'  # MU_CHANGE
     architecture: 'x64'
 
 - script: |

.azurepipelines/templates/platform-build-run-steps.yml

@@ -42,7 +42,7 @@ steps:
 
 - task: UsePythonVersion@0
   inputs:
-    versionSpec: "3.8.x"
+    versionSpec: "3.9.x"    # MU_CHANGE
     architecture: "x64"
 
 - script: pip install -r pip-requirements.txt --upgrade
@@ -51,6 +51,12 @@ steps:
 # Set default
 - bash: echo "##vso[task.setvariable variable=pkg_count]${{ 1 }}"
 
+# Fetch the target branch so that pr_eval can diff them.
+# Seems like azure pipelines/github changed checkout process in nov 2020.
+- script: git fetch origin $(System.PullRequest.targetBranch)
+  displayName: fetch target branch
+  condition: eq(variables['Build.Reason'], 'PullRequest')
+
 # trim the package list if this is a PR
 - task: CmdLine@1
   displayName: Check if ${{ parameters.build_pkg }} need testing

.azurepipelines/templates/pr-gate-build-job.yml

@@ -4,14 +4,15 @@
 # template file used to build supported packages.
 #
 # Copyright (c) Microsoft Corporation.
-# Copyright (c) 2020, ARM Limited. All rights reserved.<BR>
+# Copyright (c) 2020 - 2021, ARM Limited. All rights reserved.<BR>
 # SPDX-License-Identifier: BSD-2-Clause-Patent
 ##
 
 parameters:
   tool_chain_tag: ''
   vm_image: ''
   arch_list: ''
+  extra_steps: []   # MU_CHANGE
 
 # Build step
 jobs:
@@ -40,6 +41,7 @@ jobs:
     vmImage: ${{ parameters.vm_image }}
 
   steps:
+  - ${{ parameters.extra_steps }}   # MU_CHANGE
   - template: pr-gate-steps.yml
     parameters:
       tool_chain_tag: ${{ parameters.tool_chain_tag }}

.azurepipelines/templates/pr-gate-steps.yml

@@ -20,7 +20,7 @@ steps:
 
 - task: UsePythonVersion@0
   inputs:
-    versionSpec: '3.8.x'
+    versionSpec: '3.9.x'    # MU_CHANGE
     architecture: 'x64'
 
 - script: pip install -r pip-requirements.txt --upgrade
@@ -31,24 +31,18 @@ steps:
     echo "##vso[task.setvariable variable=pkgs_to_build]${{ parameters.build_pkgs }}"
     echo "##vso[task.setvariable variable=pkg_count]${{ 1 }}"
 
-# MU_CHANGE [BEGIN] - Workaround an azure pipelines bug.
-# $System.PullRequest.targetBranch looks like "dev/whatever" on GitHub,
-# but looks like "refs/heads/dev/whatever" on DevOps. The DevOps version
-# can't be used for comparison for the PR Eval.
-- powershell:
-    $TargetBranch = "$(System.PullRequest.targetBranch)".replace('refs/heads/', '');
-    Write-Host "##vso[task.setvariable variable=pr_compare_branch]origin/$TargetBranch";
-  displayName: Workaround for Branch Names
+# Fetch the target branch so that pr_eval can diff them.
+# Seems like azure pipelines/github changed checkout process in nov 2020.
+- script: git fetch origin $(System.PullRequest.targetBranch)
+  displayName: fetch target branch
   condition: eq(variables['Build.Reason'], 'PullRequest')
-# MU_CHANGE [END]
 
 # trim the package list if this is a PR
 - task: CmdLine@1
   displayName: Check if ${{ parameters.build_pkgs }} need testing
   inputs:
     filename: stuart_pr_eval
-    # MU_CHANGE - Workaround an azure pipelines bug.
-    arguments: -c .pytool/CISettings.py -p ${{ parameters.build_pkgs }} --pr-target $(pr_compare_branch) --output-csv-format-string "##vso[task.setvariable variable=pkgs_to_build;isOutpout=true]{pkgcsv}" --output-count-format-string "##vso[task.setvariable variable=pkg_count;isOutpout=true]{pkgcount}"
+    arguments: -c .pytool/CISettings.py -p ${{ parameters.build_pkgs }} --pr-target origin/$(System.PullRequest.targetBranch) --output-csv-format-string "##vso[task.setvariable variable=pkgs_to_build;isOutpout=true]{pkgcsv}" --output-count-format-string "##vso[task.setvariable variable=pkg_count;isOutpout=true]{pkgcount}"
   condition: eq(variables['Build.Reason'], 'PullRequest')
 
 # install spell check prereqs

.azurepipelines/templates/spell-check-prereq-steps.yml

@@ -13,7 +13,7 @@ parameters:
 steps:
 - task: NodeTool@0
   inputs:
-    versionSpec: '12.x'  # mu_change [TCBZ2618]
+    versionSpec: '14.x'  # MU_CHANGE
     #checkLatest: false # Optional
   condition: and(gt(variables.pkg_count, 0), succeeded())
 

.gitignore

@@ -8,9 +8,11 @@ tags/
 *.bak
 BuildConfig.conf
 
+# Repo-specific
+_TEMP_*/
+/Conf/
+
 # Ignore cloned dependencies
 /Common
 /Silicon
 /MU_BASECORE
-
-_TEMP_*/

.pytool/CISettings.py

@@ -21,16 +21,24 @@ def __init__(self):
         self.ActualTargets = []
         self.ActualArchitectures = []
         self.ActualToolChainTag = ""
+        self.UseBuiltInBaseTools = None
+        self.ActualScopes = None
 
     # ####################################################################################### #
     #                             Extra CmdLine configuration                                 #
     # ####################################################################################### #
 
     def AddCommandLineOptions(self, parserObj):
-        pass
+        group = parserObj.add_mutually_exclusive_group()
+        group.add_argument("-force_piptools", "--fpt", dest="force_piptools", action="store_true", default=False, help="Force the system to use pip tools")
+        group.add_argument("-no_piptools", "--npt", dest="no_piptools", action="store_true", default=False, help="Force the system to not use pip tools")
 
     def RetrieveCommandLineOptions(self, args):
-        pass
+        super().RetrieveCommandLineOptions(args)
+        if args.force_piptools:
+            self.UseBuiltInBaseTools = True
+        if args.no_piptools:
+            self.UseBuiltInBaseTools = False
 
     # ####################################################################################### #
     #                        Default Support for this Ci Build                                #
@@ -120,19 +128,38 @@ def SetTargets(self, list_of_requested_target):
 
     def GetActiveScopes(self):
         ''' return tuple containing scopes that should be active for this process '''
-        scopes = ("cibuild", "edk2-build", "host-based-test")
-
-        self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
-
-        if GetHostInfo().os.upper() == "LINUX" and self.ActualToolChainTag.upper().startswith("GCC"):
-            if "AARCH64" in self.ActualArchitectures:
-                scopes += ("gcc_aarch64_linux",)
-            if "ARM" in self.ActualArchitectures:
-                scopes += ("gcc_arm_linux",)
-            if "RISCV64" in self.ActualArchitectures:
-                scopes += ("gcc_riscv64_unknown",)
-
-        return scopes
+        if self.ActualScopes is None:
+            scopes = ("cibuild", "edk2-build", "host-based-test")
+
+            self.ActualToolChainTag = shell_environment.GetBuildVars().GetValue("TOOL_CHAIN_TAG", "")
+
+            is_linux = GetHostInfo().os.upper() == "LINUX"
+
+            if self.UseBuiltInBaseTools is None:
+                is_linux = GetHostInfo().os.upper() == "LINUX"
+                # try and import the pip module for basetools
+                try:
+                    import edk2basetools
+                    self.UseBuiltInBaseTools = True
+                except ImportError:
+                    self.UseBuiltInBaseTools = False
+                    pass
+
+            if self.UseBuiltInBaseTools == True:
+                scopes += ('pipbuild-unix',) if is_linux else ('pipbuild-win',)
+                logging.warning("Using Pip Tools based BaseTools")
+            else:
+                logging.warning("Falling back to using in-tree BaseTools")
+
+            if is_linux and self.ActualToolChainTag.upper().startswith("GCC"):
+                if "AARCH64" in self.ActualArchitectures:
+                    scopes += ("gcc_aarch64_linux",)
+                if "ARM" in self.ActualArchitectures:
+                    scopes += ("gcc_arm_linux",)
+                if "RISCV64" in self.ActualArchitectures:
+                    scopes += ("gcc_riscv64_unknown",)
+            self.ActualScopes = scopes
+        return self.ActualScopes
 
     def GetRequiredSubmodules(self):
         ''' return iterable containing RequiredSubmodule objects.

pip-requirements.txt

@@ -12,5 +12,7 @@
 # https://www.python.org/dev/peps/pep-0440/#version-specifiers
 ##
 
-edk2-pytool-library~=0.10.8 # MU_CHANGE - update to 0.10.8 or later
-edk2-pytool-extensions~=0.13.3
+edk2-pytool-library~=0.10.13 # MU_CHANGE - update to 0.10.13 or later
+edk2-pytool-extensions~=0.14.0 # MU_CHANGE - update to 0.14.0 or later
+edk2-basetools==0.1.2
+antlr4-python3-runtime==4.7.1