-
Notifications
You must be signed in to change notification settings - Fork 134
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[MDLINT_PLUGIN] Add MarkdownLint checker as a CI Plugin
- Loading branch information
1 parent
e23b836
commit 90fb2a0
Showing
8 changed files
with
448 additions
and
5 deletions.
There are no files selected for viewing
18 changes: 18 additions & 0 deletions
18
.azurepipelines/templates/markdownlint-check-prereq-steps.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
## @file | ||
# File templates/markdownlint-check-prereq-steps.yml | ||
# | ||
# template file used to install markdownlint checking prerequisite | ||
# depends on spell-check-prereq-steps to run first to set the node version | ||
# | ||
# Copyright (c) Microsoft Corporation. | ||
# SPDX-License-Identifier: BSD-2-Clause-Patent | ||
## | ||
|
||
parameters: | ||
none: '' | ||
|
||
steps: | ||
|
||
- script: npm install -g [email protected] | ||
displayName: "Install markdown linter" | ||
condition: and(gt(variables.pkg_count, 0), succeeded()) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,154 @@ | ||
## @file | ||
# File templates/pr-gate-steps.yml | ||
# | ||
# template file containing the steps to build | ||
# | ||
# Copyright (c) Microsoft Corporation. | ||
# SPDX-License-Identifier: BSD-2-Clause-Patent | ||
## | ||
|
||
parameters: | ||
tool_chain_tag: '' | ||
build_pkgs: '' | ||
build_targets: '' | ||
build_archs: '' | ||
usePythonVersion: '' | ||
extra_install_step: [] | ||
|
||
steps: | ||
- bash: | | ||
echo "##vso[task.prependpath]${HOME}/.local/bin" | ||
echo "new PATH=${PATH}" | ||
displayName: Set PATH | ||
condition: eq('${{ parameters.tool_chain_tag }}', 'GCC5') | ||
|
||
- checkout: self | ||
clean: true | ||
fetchDepth: 1 | ||
|
||
- task: UsePythonVersion@0 | ||
inputs: | ||
versionSpec: ${{ parameters.usePythonVersion }} | ||
architecture: "x64" | ||
condition: ne('${{ parameters.usePythonVersion }}', '') | ||
|
||
- script: pip install -r pip-requirements.txt --upgrade | ||
displayName: 'Install/Upgrade pip modules' | ||
|
||
# Set default | ||
- bash: | | ||
echo "##vso[task.setvariable variable=pkgs_to_build]${{ parameters.build_pkgs }}" | ||
echo "##vso[task.setvariable variable=pkg_count]${{ 1 }}" | ||
# Fetch the target branch so that pr_eval can diff them. | ||
# Seems like azure pipelines/github changed checkout process in nov 2020. | ||
- script: git fetch origin $(System.PullRequest.targetBranch) | ||
displayName: fetch target branch | ||
condition: eq(variables['Build.Reason'], 'PullRequest') | ||
|
||
- ${{ parameters.extra_install_step }} | ||
|
||
# trim the package list if this is a PR | ||
- task: CmdLine@1 | ||
displayName: Check if ${{ parameters.build_pkgs }} need testing | ||
inputs: | ||
filename: stuart_pr_eval | ||
arguments: -c .pytool/CISettings.py -p ${{ parameters.build_pkgs }} --pr-target origin/$(System.PullRequest.targetBranch) --output-csv-format-string "##vso[task.setvariable variable=pkgs_to_build;isOutpout=true]{pkgcsv}" --output-count-format-string "##vso[task.setvariable variable=pkg_count;isOutpout=true]{pkgcount}" | ||
condition: eq(variables['Build.Reason'], 'PullRequest') | ||
|
||
# install spell check prereqs | ||
- template: spell-check-prereq-steps.yml | ||
|
||
# MU_CHANGE - Add MarkdownLint | ||
# install markdownlint check prereqs | ||
- template: markdownlint-check-prereq-steps.yml | ||
|
||
# Build repo | ||
- task: CmdLine@1 | ||
displayName: Setup ${{ parameters.build_pkgs }} ${{ parameters.build_archs}} | ||
inputs: | ||
filename: stuart_setup | ||
arguments: -c .pytool/CISettings.py -p $(pkgs_to_build) -t ${{ parameters.build_targets}} -a ${{ parameters.build_archs}} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}} | ||
condition: and(gt(variables.pkg_count, 0), succeeded()) | ||
|
||
- task: CmdLine@1 | ||
displayName: Update ${{ parameters.build_pkgs }} ${{ parameters.build_archs}} | ||
inputs: | ||
filename: stuart_update | ||
arguments: -c .pytool/CISettings.py -p $(pkgs_to_build) -t ${{ parameters.build_targets}} -a ${{ parameters.build_archs}} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}} | ||
condition: and(gt(variables.pkg_count, 0), succeeded()) | ||
|
||
# build basetools | ||
# do this after setup and update so that code base dependencies | ||
# are all resolved. | ||
- template: basetools-build-steps.yml | ||
parameters: | ||
tool_chain_tag: ${{ parameters.tool_chain_tag }} | ||
|
||
- task: CmdLine@1 | ||
displayName: Build and Test ${{ parameters.build_pkgs }} ${{ parameters.build_archs}} | ||
inputs: | ||
filename: stuart_ci_build | ||
arguments: -c .pytool/CISettings.py -p $(pkgs_to_build) -t ${{ parameters.build_targets}} -a ${{ parameters.build_archs}} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}} | ||
condition: and(gt(variables.pkg_count, 0), succeeded()) | ||
|
||
# Publish Test Results to Azure Pipelines/TFS | ||
- task: PublishTestResults@2 | ||
displayName: 'Publish junit test results' | ||
continueOnError: true | ||
condition: and( succeededOrFailed(),gt(variables.pkg_count, 0)) | ||
inputs: | ||
testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit | ||
testResultsFiles: 'Build/TestSuites.xml' | ||
#searchFolder: '$(System.DefaultWorkingDirectory)' # Optional | ||
mergeTestResults: true # Optional | ||
testRunTitle: $(System.JobName) # Optional | ||
#buildPlatform: # Optional | ||
#buildConfiguration: # Optional | ||
publishRunAttachments: true # Optional | ||
|
||
# Publish Test Results to Azure Pipelines/TFS | ||
- task: PublishTestResults@2 | ||
displayName: 'Publish host based test results for $(System.JobName)' | ||
continueOnError: true | ||
condition: and( succeededOrFailed(), gt(variables.pkg_count, 0)) | ||
inputs: | ||
testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit | ||
testResultsFiles: 'Build/**/*.result.xml' | ||
#searchFolder: '$(System.DefaultWorkingDirectory)' # Optional | ||
mergeTestResults: false # Optional | ||
testRunTitle: ${{ parameters.build_pkgs }} # Optional | ||
#buildPlatform: # Optional | ||
#buildConfiguration: # Optional | ||
publishRunAttachments: true # Optional | ||
|
||
# Copy the build logs to the artifact staging directory | ||
- task: CopyFiles@2 | ||
displayName: "Copy build logs" | ||
inputs: | ||
targetFolder: '$(Build.ArtifactStagingDirectory)' | ||
SourceFolder: 'Build' | ||
contents: | | ||
BUILDLOG_*.txt | ||
BUILDLOG_*.md | ||
CI_*.txt | ||
CI_*.md | ||
CISETUP.txt | ||
SETUPLOG.txt | ||
UPDATE_LOG.txt | ||
PREVALLOG.txt | ||
TestSuites.xml | ||
**/BUILD_TOOLS_REPORT.html | ||
**/OVERRIDELOG.TXT | ||
coverage.xml | ||
flattenFolders: true | ||
condition: succeededOrFailed() | ||
|
||
# Publish build artifacts to Azure Artifacts/TFS or a file share | ||
- task: PublishBuildArtifacts@1 | ||
continueOnError: true | ||
displayName: "Publish build logs" | ||
inputs: | ||
pathtoPublish: '$(Build.ArtifactStagingDirectory)' | ||
artifactName: 'Build Logs $(System.JobName)' | ||
condition: succeededOrFailed() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,177 @@ | ||
# @file MarkdownLintCheck.py | ||
# | ||
# An edk2-pytool based plugin wrapper for markdownlint | ||
# | ||
# Copyright (c) Microsoft Corporation. | ||
# SPDX-License-Identifier: BSD-2-Clause-Patent | ||
## | ||
import logging | ||
import json | ||
import yaml | ||
from io import StringIO | ||
import os | ||
from typing import List | ||
from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin | ||
from edk2toollib.utility_functions import RunCmd | ||
from edk2toolext.environment.var_dict import VarDict | ||
from edk2toolext.environment import version_aggregator | ||
|
||
|
||
class MarkdownLintCheck(ICiBuildPlugin): | ||
""" | ||
A CiBuildPlugin that uses the markdownlint-cli node module to scan the files | ||
from the package being tested for linter errors. | ||
The linter config file (.markdownlint.yaml) must be present in one of the defined | ||
locations otherwise the test will be skipped. These locations were picked to also | ||
align with how editors and tools will find the config file. | ||
1st priority location - At the package root | ||
2nd Priority location - At the workspace root of the build (suggested location unless override needed) | ||
Configuration options: | ||
"MarkdownLintCheck": { | ||
"AuditOnly": False, # If True, log all errors and then mark as skipped | ||
"IgnoreFiles": [] # package root relative file, folder, or glob pattern to ignore | ||
} | ||
""" | ||
|
||
CONFIG_FILE_NAME = ".markdownlint.yaml" | ||
|
||
def GetTestName(self, packagename: str, environment: VarDict) -> tuple: | ||
""" Provide the testcase name and classname for use in reporting | ||
Args: | ||
packagename: string containing name of package to build | ||
environment: The VarDict for the test to run in | ||
Returns: | ||
a tuple containing the testcase name and the classname | ||
(testcasename, classname) | ||
testclassname: a descriptive string for the testcase can include whitespace | ||
classname: should be patterned <packagename>.<plugin>.<optionally any unique condition> | ||
""" | ||
return ("Lint Markdown files in " + packagename, packagename + ".markdownlint") | ||
|
||
## | ||
# External function of plugin. This function is used to perform the task of the CiBuild Plugin | ||
# | ||
# - package is the edk2 path to package. This means workspace/packagepath relative. | ||
# - edk2path object configured with workspace and packages path | ||
# - PkgConfig Object (dict) for the pkg | ||
# - EnvConfig Object | ||
# - Plugin Manager Instance | ||
# - Plugin Helper Obj Instance | ||
# - Junit Logger | ||
# - output_stream the StringIO output stream from this plugin via logging | ||
|
||
def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None): | ||
Errors = [] | ||
|
||
abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSystemFromEdk2RelativePath( | ||
packagename) | ||
|
||
if abs_pkg_path is None: | ||
tc.SetSkipped() | ||
tc.LogStdError("No package {0}".format(packagename)) | ||
return -1 | ||
|
||
# check for node | ||
return_buffer = StringIO() | ||
ret = RunCmd("node", "--version", outstream=return_buffer) | ||
if (ret != 0): | ||
tc.SetSkipped() | ||
tc.LogStdError("NodeJs not installed. Test can't run") | ||
logging.warning("NodeJs not installed. Test can't run") | ||
return -1 | ||
node_version = return_buffer.getvalue().strip() # format vXX.XX.XX | ||
tc.LogStdOut(f"Node version: {node_version}") | ||
|
||
# Check for markdownlint-cli | ||
return_buffer = StringIO() | ||
ret = RunCmd("markdownlint", "--version", outstream=return_buffer) | ||
if (ret != 0): | ||
tc.SetSkipped() | ||
tc.LogStdError("markdownlint not installed. Test can't run") | ||
logging.warning("markdownlint not installed. Test can't run") | ||
return -1 | ||
mdl_version = return_buffer.getvalue().strip() # format XX.XX.XX | ||
tc.LogStdOut(f"MarkdownLint version: {mdl_version}") | ||
version_aggregator.GetVersionAggregator().ReportVersion( | ||
"MarkDownLint", mdl_version, version_aggregator.VersionTypes.INFO) | ||
|
||
# Get relative path for the root of package to use with ignore and path parameters | ||
relpath = os.path.relpath(abs_pkg_path) | ||
|
||
# Newer versions of markdownlint don't understand backslashes | ||
relpath = relpath.replace(os.path.sep, '/') | ||
|
||
# | ||
# check for any package specific ignore patterns defined by package config | ||
# | ||
Ignores = [] | ||
if("IgnoreFiles" in pkgconfig): | ||
for i in pkgconfig["IgnoreFiles"]: | ||
Ignores.append(f"{relpath}/{i}") | ||
|
||
# | ||
# Make the path string to check | ||
# | ||
path_to_check = f'{relpath}/**/*.md' | ||
|
||
# get path to config file - | ||
|
||
# Currently there is support for two different config files | ||
# If the config file is not found then the test case is skipped | ||
# | ||
# 1st - At the package root | ||
# 2nd - At the workspace root of the build | ||
config_file_path = None | ||
|
||
# 1st check to see if the config file is at package root | ||
if os.path.isfile(os.path.join(abs_pkg_path, MarkdownLintCheck.CONFIG_FILE_NAME)): | ||
config_file_path = os.path.join(abs_pkg_path, MarkdownLintCheck.CONFIG_FILE_NAME) | ||
|
||
# 2nd check to see if at workspace root | ||
elif os.path.isfile(os.path.join(Edk2pathObj.WorkspacePath, MarkdownLintCheck.CONFIG_FILE_NAME)): | ||
config_file_path = os.path.join(Edk2pathObj.WorkspacePath, MarkdownLintCheck.CONFIG_FILE_NAME) | ||
|
||
# If not found - skip test | ||
else: | ||
tc.SetSkipped() | ||
tc.LogStdError(f"{MarkdownLintCheck.CONFIG_FILE_NAME} not found. Skipping test") | ||
logging.warning(f"{MarkdownLintCheck.CONFIG_FILE_NAME} not found. Skipping test") | ||
return -1 | ||
|
||
|
||
# Run the linter | ||
results = self._check_markdown(path_to_check, config_file_path, Ignores) | ||
for r in results: | ||
tc.LogStdError(r.strip()) | ||
|
||
# add result to test case | ||
overall_status = len(results) | ||
if overall_status != 0: | ||
if "AuditOnly" in pkgconfig and pkgconfig["AuditOnly"]: | ||
# set as skipped if AuditOnly | ||
tc.SetSkipped() | ||
return -1 | ||
else: | ||
tc.SetFailed("Markdown Lint Check {0} Failed. Errors {1}".format( | ||
packagename, overall_status), "CHECK_FAILED") | ||
else: | ||
tc.SetSuccess() | ||
return overall_status | ||
|
||
def _check_markdown(self, rel_file_to_check: os.PathLike, abs_config_file_to_use: os.PathLike, Ignores: List) -> []: | ||
output = StringIO() | ||
param = f"--config {abs_config_file_to_use}" | ||
for a in Ignores: | ||
param += f' --ignore "{a}"' | ||
param += f' "{rel_file_to_check}"' | ||
|
||
ret = RunCmd( | ||
"markdownlint", param, outstream=output) | ||
if ret == 0: | ||
return [] | ||
else: | ||
return output.getvalue().strip().splitlines() |
11 changes: 11 additions & 0 deletions
11
.pytool/Plugin/MarkdownLintCheck/MarkdownLintCheck_plug_in.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
## @file | ||
# CiBuildPlugin used to lint repository documentation in markdown files | ||
# | ||
# Copyright (c) Microsoft Corporation. | ||
# SPDX-License-Identifier: BSD-2-Clause-Patent | ||
## | ||
{ | ||
"scope": "cibuild", | ||
"name": "Markdown Lint Test", | ||
"module": "MarkdownLintCheck" | ||
} |
Oops, something went wrong.