Add Line Ending plugin (#1150)

## Description

Cherry-pick the LineEndingCheck plugin in its entirety including the
various bug fixes and performance improvement fixes it has received over
time. Additionally, all line endings for every file were updated to be
CRLF to pass the new pipeline check.

The commits included from 202311:
8080124
6ddc3a5
59f3382
24a4f42
418f07b

These commits do the following:

Revert the temporary commit disabling the ARM and ARM64 pipeline builds
for CI and Basetools. This ensures that all normal Mu CI is run after
this point.
---
Fixed a bug where the plugin would fail if not run from the projects
root directory.
---
Improves performance for the plugin compared to its initial release
---

For each item, place an "x" in between `[` and `]` if true. Example:
`[x]`.
_(you can also check items in the GitHub UI)_

- [ ] Impacts functionality?
- **Functionality** - Does the change ultimately impact how firmware
functions?
- Examples: Add a new library, publish a new PPI, update an algorithm,
...
- [ ] Impacts security?
- **Security** - Does the change have a direct security impact on an
application,
    flow, or firmware?
  - Examples: Crypto algorithm change, buffer overflow fix, parameter
    validation improvement, ...
- [ ] Breaking change?
- **Breaking change** - Will anyone consuming this change experience a
break
    in build or boot behavior?
- Examples: Add a new library class, move a module to a different repo,
call
    a function in a new library class in a pre-existing module, ...
- [ ] Includes tests?
  - **Tests** - Does the change include any explicit test code?
  - Examples: Unit tests, integration tests, robot tests, ...
- [ ] Includes documentation?
- **Documentation** - Does the change contain explicit documentation
additions
    outside direct code modifications (and comments)?
- Examples: Update readme file, add feature readme file, link to
documentation
    on an a separate Web page, ...

## How This Was Tested

N/A

## Integration Instructions

N/A

---------

Signed-off-by: Michael Kubacki <[email protected]>
Co-authored-by: Michael Kubacki <[email protected]>

.pytool/Plugin/LineEndingCheck/LineEndingCheck.py

@@ -0,0 +1,324 @@
+# @file LineEndingCheck.py
+#
+# An edk2-pytool based plugin that checks line endings.
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+
+import glob
+from io import StringIO
+import logging
+import os
+import shutil
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Tuple
+
+from edk2toolext.environment.plugin_manager import PluginManager
+from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
+from edk2toolext.environment.plugintypes.uefi_helper_plugin import \
+    HelperFunctions
+from edk2toolext.environment.var_dict import VarDict
+from edk2toollib.gitignore_parser import parse_gitignore_lines
+from edk2toollib.log.junit_report_format import JunitReportTestCase
+from edk2toollib.uefi.edk2.path_utilities import Edk2Path
+from edk2toollib.utility_functions import RunCmd
+from git import Repo
+
+
+PLUGIN_NAME = "LineEndingCheck"
+
+LINE_ENDINGS = [
+    b'\r\n',
+    b'\n\r',
+    b'\n',
+    b'\r'
+]
+
+ALLOWED_LINE_ENDING = b'\r\n'
+
+#
+# Based on a solution for binary file detection presented in
+# https://stackoverflow.com/a/7392391.
+#
+_TEXT_CHARS = bytearray(
+    {7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})
+
+
+def _is_binary_string(_bytes: bytes) -> bool:
+    return bool(_bytes.translate(None, _TEXT_CHARS))
+
+
+class LineEndingCheckBadLineEnding(Exception):
+    pass
+
+class LineEndingCheckGitIgnoreFileException(Exception):
+    pass
+
+class LineEndingCheck(ICiBuildPlugin):
+    """
+    A CiBuildPlugin that checks whether line endings are a certain format.
+
+    By default, the plugin runs against all files in a package unless a
+    specific file or file extension is excluded.
+
+    Configuration options:
+    "LineEndingCheck": {
+        "IgnoreFiles": [],            # File patterns to ignore.
+    }
+    """
+
+    def GetTestName(self, packagename: str, environment: VarDict) -> Tuple:
+        """ Provide the testcase name and classname for use in reporting
+
+            Args:
+              packagename: String containing name of package to build.
+              environment: The VarDict for the test to run in.
+
+            Returns:
+                A tuple containing the testcase name and the classname
+                (testcasename, classname)
+                testclassname: a descriptive string for the testcase can
+                               include whitespace
+                classname: Should be patterned <packagename>.<plugin>
+                           .<optionally any unique condition>
+        """
+        return ("Check line endings in " + packagename, packagename +
+                "." + PLUGIN_NAME)
+
+    # Note: This function access git via the command line
+    #
+    #   function to check and warn if git config reports that
+    #   autocrlf is configured to TRUE
+    def _check_autocrlf(self):
+        r = Repo(self._abs_workspace_path)
+        try:
+            result = r.config_reader().get_value("core", "autocrlf")
+            if result:
+                logging.warning(f"git config core.autocrlf is set to {result} "
+                                f"recommended setting is false "
+                                f"git config --global core.autocrlf false")
+        except Exception:
+            logging.warning(f"git config core.autocrlf is not set "
+                            f"recommended setting is false "
+                            f"git config --global core.autocrlf false")
+        return
+
+    # Note: This function currently accesses git via the git command to prevent
+    #       introducing a new Python git module dependency in mu_basecore
+    #       on gitpython.
+    #
+    #       After gitpython is adopted by edk2-pytool-extensions, this
+    #       implementation can be updated to use the gitpython interface.
+    def _get_git_ignored_paths(self) -> List[Path]:
+        """"
+        Gets paths ignored by git.
+
+        Returns:
+            List[str]: A list of file absolute path strings to all files
+            ignored in this git repository.
+
+            If git is not found, an empty list will be returned.
+        """
+        if not shutil.which("git"):
+            logging.warning(
+                "Git is not found on this system. Git submodule paths will "
+                "not be considered.")
+            return []
+
+        outstream_buffer = StringIO()
+        exit_code = RunCmd("git", "ls-files --other",
+                           workingdir=self._abs_workspace_path,
+                           outstream=outstream_buffer,
+                           logging_level=logging.NOTSET)
+        if (exit_code != 0):
+            raise LineEndingCheckGitIgnoreFileException(
+                f"An error occurred reading git ignore settings. This will "
+                f"prevent LineEndingCheck from running against the expected "
+                f"set of files.")
+
+        # Note: This will potentially be a large list, but at least sorted
+        rel_paths = outstream_buffer.getvalue().strip().splitlines()
+        abs_paths = []
+        for path in rel_paths:
+            abs_paths.append(Path(
+                os.path.normpath(os.path.join(self._abs_workspace_path, path))))
+        return abs_paths
+
+    # Note: This function currently accesses git via the git command to prevent
+    #       introducing a new Python git module dependency in mu_basecore
+    #       on gitpython.
+    #
+    #       After gitpython is adopted by edk2-pytool-extensions, this
+    #       implementation can be updated to use the gitpython interface.
+    def _get_git_submodule_paths(self) -> List[Path]:
+        """
+        Gets submodule paths recognized by git.
+
+        Returns:
+            List[str]: A list of directory absolute path strings to the root
+            of each submodule in the workspace repository.
+
+            If git is not found, an empty list will be returned.
+        """
+        if not shutil.which("git"):
+            logging.warning(
+                "Git is not found on this system. Git submodule paths will "
+                "not be considered.")
+            return []
+
+        if os.path.isfile(os.path.join(self._abs_workspace_path, ".gitmodules")):
+            logging.info(
+                ".gitmodules file found. Excluding submodules in "
+                "LineEndingCheck.")
+
+            outstream_buffer = StringIO()
+            exit_code = RunCmd("git",
+                            "config --file .gitmodules --get-regexp path",
+                            workingdir=self._abs_workspace_path,
+                            outstream=outstream_buffer,
+                            logging_level=logging.NOTSET)
+            if (exit_code != 0):
+                raise LineEndingCheckGitIgnoreFileException(
+                    f".gitmodule file detected but an error occurred reading "
+                    f"the file. Cannot proceed with unknown submodule paths.")
+
+            submodule_paths = []
+            for line in outstream_buffer.getvalue().strip().splitlines():
+                submodule_paths.append(Path(
+                    os.path.normpath(os.path.join(self._abs_workspace_path, line.split()[1]))))
+
+            return submodule_paths
+        else:
+            return []
+
+    def _get_files_ignored_in_config(self,
+                                     pkg_config: Dict[str, List[str]],
+                                     base_dir: str) -> Callable[[str], bool]:
+        """"
+        Returns a function that returns true if a given file string path is
+        ignored in the plugin configuration file and false otherwise.
+
+        Args:
+            pkg_config: Dictionary with the package configuration
+            base_dir: Base directory of the package
+
+        Returns:
+            Callable[[None], None]: A test case function.
+        """
+        ignored_files = []
+        if pkg_config.get("IgnoreFilesWithNoExtension", False):
+            ignored_files.extend(['*', '!*.*', '!*/'])
+        if "IgnoreFiles" in pkg_config:
+            ignored_files.extend(pkg_config["IgnoreFiles"])
+
+        # Pass "Package configuration file" as the source file path since
+        # the actual configuration file name is unknown to this plugin and
+        # this provides a generic description of the file that provided
+        # the ignore file content.
+        #
+        # This information is only used for reporting (not used here) and
+        # the ignore lines are being passed directly as they are given to
+        # this plugin.
+        return parse_gitignore_lines(ignored_files,
+                                     "Package configuration file",
+                                     base_dir)
+
+    def RunBuildPlugin(self, package_rel_path: str, edk2_path: Edk2Path,
+                       package_config: Dict[str, List[str]],
+                       environment_config: Any,
+                       plugin_manager: PluginManager,
+                       plugin_manager_helper: HelperFunctions,
+                       tc: JunitReportTestCase, output_stream=None) -> int:
+        """
+        External function of plugin. This function is used to perform the task
+        of the CiBuild Plugin.
+
+        Args:
+          - package_rel_path: edk2 workspace relative path to the package
+          - edk2_path: Edk2Path object with workspace and packages paths
+          - package_config: Dictionary with the package configuration
+          - environment_config: Environment configuration
+          - plugin_manager: Plugin Manager Instance
+          - plugin_manager_helper: Plugin Manager Helper Instance
+          - tc: JUnit test case
+          - output_stream: The StringIO output stream from this plugin
+            (logging)
+
+        Returns:
+          >0 : Number of errors found
+          0  : Ran successfully
+          -1 : Skipped due to a missing pre-requisite
+        """
+        self._abs_workspace_path = edk2_path.WorkspacePath
+        self._check_autocrlf()
+        self._abs_pkg_path = \
+            edk2_path.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
+                        package_rel_path)
+
+        if self._abs_pkg_path is None:
+            tc.SetSkipped()
+            tc.LogStdError(f"Package folder not found {self._abs_pkg_path}")
+            return 0
+
+        ignore_files = set(self._get_git_ignored_paths())
+        ignore_dirs = set(self._get_git_submodule_paths())
+        ignore_filter = self._get_files_ignored_in_config(package_config, self._abs_pkg_path)
+
+        file_count = 0
+        line_ending_count = dict.fromkeys(LINE_ENDINGS, 0)
+        for file in Path(self._abs_pkg_path).rglob('*'):
+            if file.is_dir():
+                continue
+
+            if any(file.is_relative_to(ignore_dir) for ignore_dir in ignore_dirs):
+                continue
+
+            if ignore_filter(file):
+                continue
+
+            if file in ignore_files:
+                continue
+
+            with open(file.resolve(), 'rb') as fb:
+                if not fb.readable() or _is_binary_string(fb.read(1024)):
+                    continue
+                fb.seek(0)
+
+                for lineno, line in enumerate(fb):
+                    try:
+                        for e in LINE_ENDINGS:
+                            if line.endswith(e):
+                                line_ending_count[e] += 1
+
+                                if e is not ALLOWED_LINE_ENDING:
+                                    file_path = file.relative_to(
+                                        Path(self._abs_workspace_path)).as_posix()
+                                    file_count += 1
+
+                                    tc.LogStdError(
+                                        f"Line ending on Line {lineno} in "
+                                        f"{file_path} is not allowed.\nLine "
+                                        f"ending is {e} and should be "
+                                        f"{ALLOWED_LINE_ENDING}.")
+                                    logging.error(
+                                        f"Line ending on Line {lineno} in "
+                                        f"{file_path} is not allowed.\nLine "
+                                        f"ending is {e} and should be "
+                                        f"{ALLOWED_LINE_ENDING}.")
+                                    raise LineEndingCheckBadLineEnding
+                                break
+                    except LineEndingCheckBadLineEnding:
+                        break
+
+        del line_ending_count[ALLOWED_LINE_ENDING]
+
+        if any(line_ending_count.values()):
+            tc.SetFailed(
+                f"{PLUGIN_NAME} failed due to {file_count} files with "
+                f"incorrect line endings.",
+                "CHECK_FAILED")
+        else:
+            tc.SetSuccess()
+
+        return sum(line_ending_count.values())

.pytool/Plugin/LineEndingCheck/Readme.md

@@ -0,0 +1,33 @@
+# Line Ending Check Plugin
+
+This CiBuildPlugin scans all the files in a package to verify that the line endings are CRLF.
+
+> _Note:_ If you encounter a line ending issue found by this plugin, update your development environment to avoid
+> issues again in the future.
+>
+> Most problems are caused by `autocrlf=true` in git settings, which will automatically adjust line endings upon
+> checkout and commit which distorts the actual line endings from being consistent locally and remotely. In
+> other cases, developing within a Linux workspace will natively use LF by default.
+>
+> It is simplest to set `autocrlf=false` to prevent manipulation of line endings outside of the actual values and set
+> up your editor to use CRLF line endings within the project.
+
+## Configuration
+
+The plugin can be configured to ignore certain files.
+
+``` yaml
+"LineEndingCheck": {
+    "IgnoreFiles": []
+    "IgnoreFilesWithNoExtension": False
+}
+```
+
+### IgnoreFiles
+
+An **optional** list of git ignore patterns relative to the package root used to exclude files from being checked.
+
+### IgnoreFilesWithNoExtension
+
+An **optional** value that, if True, will insert the gitignore rules necessary to have this check ignore files
+that do not contain a file extension. Necessary for binary files and/or POSIX like executables.

.pytool/Plugin/LineEndingCheck/line_ending_check_plug_in.yaml

@@ -0,0 +1,11 @@
+## @file
+# CiBuildPlugin used to check line ending format.
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+##
+{
+  "scope": "cibuild",
+  "name": "Line Ending Check Test",
+  "module": "LineEndingCheck"
+}

.pytool/Plugin/UncrustifyCheck/UncrustifyCheck.py

@@ -12,7 +12,6 @@
 import os
 import pathlib
 import shutil
-import stat
 import timeit
 from edk2toolext.environment import version_aggregator
 from edk2toolext.environment.plugin_manager import PluginManager
@@ -495,7 +494,6 @@ def _initialize_file_to_format_info(self) -> None:
         for path in rel_file_paths_to_format:
             self._abs_file_paths_to_format.extend(
                 [str(path.resolve()) for path in pathlib.Path(self._abs_package_path).rglob(path)])
-
         # Remove files ignore in the plugin configuration file
         plugin_ignored_files = list(filter(self._get_files_ignored_in_config(), self._abs_file_paths_to_format))
 
@@ -641,7 +639,7 @@ def _remove_readonly(func, path, _):
             """
             Private function to attempt to change permissions on file/folder being deleted.
             """
-            os.chmod(path, stat.S_IWRITE)
+            os.chmod(path, os.stat.S_IWRITE)
             func(path)
 
         for _ in range(3):  # retry up to 3 times