Skip to content

Commit

Permalink
[MDLINT_PLUGIN] Add MarkdownLint checker as a CI Plugin
Browse files Browse the repository at this point in the history
  • Loading branch information
spbrogan authored and kenlautner committed Jul 3, 2024
1 parent e23b836 commit 90fb2a0
Show file tree
Hide file tree
Showing 8 changed files with 448 additions and 5 deletions.
18 changes: 18 additions & 0 deletions .azurepipelines/templates/markdownlint-check-prereq-steps.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
## @file
# File templates/markdownlint-check-prereq-steps.yml
#
# template file used to install markdownlint checking prerequisite
# depends on spell-check-prereq-steps to run first to set the node version
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##

parameters:
none: ''

steps:

- script: npm install -g [email protected]
displayName: "Install markdown linter"
condition: and(gt(variables.pkg_count, 0), succeeded())
154 changes: 154 additions & 0 deletions .azurepipelines/templates/pr-gate-steps.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
## @file
# File templates/pr-gate-steps.yml
#
# template file containing the steps to build
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##

parameters:
tool_chain_tag: ''
build_pkgs: ''
build_targets: ''
build_archs: ''
usePythonVersion: ''
extra_install_step: []

steps:
- bash: |
echo "##vso[task.prependpath]${HOME}/.local/bin"
echo "new PATH=${PATH}"
displayName: Set PATH
condition: eq('${{ parameters.tool_chain_tag }}', 'GCC5')

- checkout: self
clean: true
fetchDepth: 1

- task: UsePythonVersion@0
inputs:
versionSpec: ${{ parameters.usePythonVersion }}
architecture: "x64"
condition: ne('${{ parameters.usePythonVersion }}', '')

- script: pip install -r pip-requirements.txt --upgrade
displayName: 'Install/Upgrade pip modules'

# Set default
- bash: |
echo "##vso[task.setvariable variable=pkgs_to_build]${{ parameters.build_pkgs }}"
echo "##vso[task.setvariable variable=pkg_count]${{ 1 }}"
# Fetch the target branch so that pr_eval can diff them.
# Seems like azure pipelines/github changed checkout process in nov 2020.
- script: git fetch origin $(System.PullRequest.targetBranch)
displayName: fetch target branch
condition: eq(variables['Build.Reason'], 'PullRequest')

- ${{ parameters.extra_install_step }}

# trim the package list if this is a PR
- task: CmdLine@1
displayName: Check if ${{ parameters.build_pkgs }} need testing
inputs:
filename: stuart_pr_eval
arguments: -c .pytool/CISettings.py -p ${{ parameters.build_pkgs }} --pr-target origin/$(System.PullRequest.targetBranch) --output-csv-format-string "##vso[task.setvariable variable=pkgs_to_build;isOutpout=true]{pkgcsv}" --output-count-format-string "##vso[task.setvariable variable=pkg_count;isOutpout=true]{pkgcount}"
condition: eq(variables['Build.Reason'], 'PullRequest')

# install spell check prereqs
- template: spell-check-prereq-steps.yml

# MU_CHANGE - Add MarkdownLint
# install markdownlint check prereqs
- template: markdownlint-check-prereq-steps.yml

# Build repo
- task: CmdLine@1
displayName: Setup ${{ parameters.build_pkgs }} ${{ parameters.build_archs}}
inputs:
filename: stuart_setup
arguments: -c .pytool/CISettings.py -p $(pkgs_to_build) -t ${{ parameters.build_targets}} -a ${{ parameters.build_archs}} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}}
condition: and(gt(variables.pkg_count, 0), succeeded())

- task: CmdLine@1
displayName: Update ${{ parameters.build_pkgs }} ${{ parameters.build_archs}}
inputs:
filename: stuart_update
arguments: -c .pytool/CISettings.py -p $(pkgs_to_build) -t ${{ parameters.build_targets}} -a ${{ parameters.build_archs}} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}}
condition: and(gt(variables.pkg_count, 0), succeeded())

# build basetools
# do this after setup and update so that code base dependencies
# are all resolved.
- template: basetools-build-steps.yml
parameters:
tool_chain_tag: ${{ parameters.tool_chain_tag }}

- task: CmdLine@1
displayName: Build and Test ${{ parameters.build_pkgs }} ${{ parameters.build_archs}}
inputs:
filename: stuart_ci_build
arguments: -c .pytool/CISettings.py -p $(pkgs_to_build) -t ${{ parameters.build_targets}} -a ${{ parameters.build_archs}} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}}
condition: and(gt(variables.pkg_count, 0), succeeded())

# Publish Test Results to Azure Pipelines/TFS
- task: PublishTestResults@2
displayName: 'Publish junit test results'
continueOnError: true
condition: and( succeededOrFailed(),gt(variables.pkg_count, 0))
inputs:
testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit
testResultsFiles: 'Build/TestSuites.xml'
#searchFolder: '$(System.DefaultWorkingDirectory)' # Optional
mergeTestResults: true # Optional
testRunTitle: $(System.JobName) # Optional
#buildPlatform: # Optional
#buildConfiguration: # Optional
publishRunAttachments: true # Optional

# Publish Test Results to Azure Pipelines/TFS
- task: PublishTestResults@2
displayName: 'Publish host based test results for $(System.JobName)'
continueOnError: true
condition: and( succeededOrFailed(), gt(variables.pkg_count, 0))
inputs:
testResultsFormat: 'JUnit' # Options: JUnit, NUnit, VSTest, xUnit
testResultsFiles: 'Build/**/*.result.xml'
#searchFolder: '$(System.DefaultWorkingDirectory)' # Optional
mergeTestResults: false # Optional
testRunTitle: ${{ parameters.build_pkgs }} # Optional
#buildPlatform: # Optional
#buildConfiguration: # Optional
publishRunAttachments: true # Optional

# Copy the build logs to the artifact staging directory
- task: CopyFiles@2
displayName: "Copy build logs"
inputs:
targetFolder: '$(Build.ArtifactStagingDirectory)'
SourceFolder: 'Build'
contents: |
BUILDLOG_*.txt
BUILDLOG_*.md
CI_*.txt
CI_*.md
CISETUP.txt
SETUPLOG.txt
UPDATE_LOG.txt
PREVALLOG.txt
TestSuites.xml
**/BUILD_TOOLS_REPORT.html
**/OVERRIDELOG.TXT
coverage.xml
flattenFolders: true
condition: succeededOrFailed()

# Publish build artifacts to Azure Artifacts/TFS or a file share
- task: PublishBuildArtifacts@1
continueOnError: true
displayName: "Publish build logs"
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: 'Build Logs $(System.JobName)'
condition: succeededOrFailed()
177 changes: 177 additions & 0 deletions .pytool/Plugin/MarkdownLintCheck/MarkdownLintCheck.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
# @file MarkdownLintCheck.py
#
# An edk2-pytool based plugin wrapper for markdownlint
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
import logging
import json
import yaml
from io import StringIO
import os
from typing import List
from edk2toolext.environment.plugintypes.ci_build_plugin import ICiBuildPlugin
from edk2toollib.utility_functions import RunCmd
from edk2toolext.environment.var_dict import VarDict
from edk2toolext.environment import version_aggregator


class MarkdownLintCheck(ICiBuildPlugin):
"""
A CiBuildPlugin that uses the markdownlint-cli node module to scan the files
from the package being tested for linter errors.
The linter config file (.markdownlint.yaml) must be present in one of the defined
locations otherwise the test will be skipped. These locations were picked to also
align with how editors and tools will find the config file.
1st priority location - At the package root
2nd Priority location - At the workspace root of the build (suggested location unless override needed)
Configuration options:
"MarkdownLintCheck": {
"AuditOnly": False, # If True, log all errors and then mark as skipped
"IgnoreFiles": [] # package root relative file, folder, or glob pattern to ignore
}
"""

CONFIG_FILE_NAME = ".markdownlint.yaml"

def GetTestName(self, packagename: str, environment: VarDict) -> tuple:
""" Provide the testcase name and classname for use in reporting
Args:
packagename: string containing name of package to build
environment: The VarDict for the test to run in
Returns:
a tuple containing the testcase name and the classname
(testcasename, classname)
testclassname: a descriptive string for the testcase can include whitespace
classname: should be patterned <packagename>.<plugin>.<optionally any unique condition>
"""
return ("Lint Markdown files in " + packagename, packagename + ".markdownlint")

##
# External function of plugin. This function is used to perform the task of the CiBuild Plugin
#
# - package is the edk2 path to package. This means workspace/packagepath relative.
# - edk2path object configured with workspace and packages path
# - PkgConfig Object (dict) for the pkg
# - EnvConfig Object
# - Plugin Manager Instance
# - Plugin Helper Obj Instance
# - Junit Logger
# - output_stream the StringIO output stream from this plugin via logging

def RunBuildPlugin(self, packagename, Edk2pathObj, pkgconfig, environment, PLM, PLMHelper, tc, output_stream=None):
Errors = []

abs_pkg_path = Edk2pathObj.GetAbsolutePathOnThisSystemFromEdk2RelativePath(
packagename)

if abs_pkg_path is None:
tc.SetSkipped()
tc.LogStdError("No package {0}".format(packagename))
return -1

# check for node
return_buffer = StringIO()
ret = RunCmd("node", "--version", outstream=return_buffer)
if (ret != 0):
tc.SetSkipped()
tc.LogStdError("NodeJs not installed. Test can't run")
logging.warning("NodeJs not installed. Test can't run")
return -1
node_version = return_buffer.getvalue().strip() # format vXX.XX.XX
tc.LogStdOut(f"Node version: {node_version}")

# Check for markdownlint-cli
return_buffer = StringIO()
ret = RunCmd("markdownlint", "--version", outstream=return_buffer)
if (ret != 0):
tc.SetSkipped()
tc.LogStdError("markdownlint not installed. Test can't run")
logging.warning("markdownlint not installed. Test can't run")
return -1
mdl_version = return_buffer.getvalue().strip() # format XX.XX.XX
tc.LogStdOut(f"MarkdownLint version: {mdl_version}")
version_aggregator.GetVersionAggregator().ReportVersion(
"MarkDownLint", mdl_version, version_aggregator.VersionTypes.INFO)

# Get relative path for the root of package to use with ignore and path parameters
relpath = os.path.relpath(abs_pkg_path)

# Newer versions of markdownlint don't understand backslashes
relpath = relpath.replace(os.path.sep, '/')

#
# check for any package specific ignore patterns defined by package config
#
Ignores = []
if("IgnoreFiles" in pkgconfig):
for i in pkgconfig["IgnoreFiles"]:
Ignores.append(f"{relpath}/{i}")

#
# Make the path string to check
#
path_to_check = f'{relpath}/**/*.md'

# get path to config file -

# Currently there is support for two different config files
# If the config file is not found then the test case is skipped
#
# 1st - At the package root
# 2nd - At the workspace root of the build
config_file_path = None

# 1st check to see if the config file is at package root
if os.path.isfile(os.path.join(abs_pkg_path, MarkdownLintCheck.CONFIG_FILE_NAME)):
config_file_path = os.path.join(abs_pkg_path, MarkdownLintCheck.CONFIG_FILE_NAME)

# 2nd check to see if at workspace root
elif os.path.isfile(os.path.join(Edk2pathObj.WorkspacePath, MarkdownLintCheck.CONFIG_FILE_NAME)):
config_file_path = os.path.join(Edk2pathObj.WorkspacePath, MarkdownLintCheck.CONFIG_FILE_NAME)

# If not found - skip test
else:
tc.SetSkipped()
tc.LogStdError(f"{MarkdownLintCheck.CONFIG_FILE_NAME} not found. Skipping test")
logging.warning(f"{MarkdownLintCheck.CONFIG_FILE_NAME} not found. Skipping test")
return -1


# Run the linter
results = self._check_markdown(path_to_check, config_file_path, Ignores)
for r in results:
tc.LogStdError(r.strip())

# add result to test case
overall_status = len(results)
if overall_status != 0:
if "AuditOnly" in pkgconfig and pkgconfig["AuditOnly"]:
# set as skipped if AuditOnly
tc.SetSkipped()
return -1
else:
tc.SetFailed("Markdown Lint Check {0} Failed. Errors {1}".format(
packagename, overall_status), "CHECK_FAILED")
else:
tc.SetSuccess()
return overall_status

def _check_markdown(self, rel_file_to_check: os.PathLike, abs_config_file_to_use: os.PathLike, Ignores: List) -> []:
output = StringIO()
param = f"--config {abs_config_file_to_use}"
for a in Ignores:
param += f' --ignore "{a}"'
param += f' "{rel_file_to_check}"'

ret = RunCmd(
"markdownlint", param, outstream=output)
if ret == 0:
return []
else:
return output.getvalue().strip().splitlines()
11 changes: 11 additions & 0 deletions .pytool/Plugin/MarkdownLintCheck/MarkdownLintCheck_plug_in.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
## @file
# CiBuildPlugin used to lint repository documentation in markdown files
#
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: BSD-2-Clause-Patent
##
{
"scope": "cibuild",
"name": "Markdown Lint Test",
"module": "MarkdownLintCheck"
}
Loading

0 comments on commit 90fb2a0

Please sign in to comment.