From 893e12df28621567e88090503718a65d0a2f70ab Mon Sep 17 00:00:00 2001 From: Michael Kubacki Date: Thu, 20 Apr 2023 13:47:27 -0400 Subject: [PATCH 1/3] .github/actions: Add initial Submodule Release Updater GitHub Action Adds an action that checks if any submodules in a repository have a GitHub release available. If so, the submodule is updated to the latest release and a pull request is made in the repository for the submodule update. Signed-off-by: Michael Kubacki --- .../submodule-release-updater/ReadMe.md | 58 +++ .../submodule-release-updater/action.yml | 425 ++++++++++++++++++ 2 files changed, 483 insertions(+) create mode 100644 .github/actions/submodule-release-updater/ReadMe.md create mode 100644 .github/actions/submodule-release-updater/action.yml diff --git a/.github/actions/submodule-release-updater/ReadMe.md b/.github/actions/submodule-release-updater/ReadMe.md new file mode 100644 index 00000000..a1fed3a2 --- /dev/null +++ b/.github/actions/submodule-release-updater/ReadMe.md @@ -0,0 +1,58 @@ +# Project Mu Submodule Release Updater GitHub Action + +This GitHub Action checks if new releases are available for submodules and creates pull requests to update +them. A single pull request is opened per submodule. At this time, the action should only be used within +Project Mu repositories. + +## How to Use + +1. Create a GitHub workflow in a repository +2. Add this GitHub Action as a step to the workflow +3. Configure the workflow to trigger as desired + - It is recommended to trigger the workflow on a schedule (e.g. daily) to check for new releases. + +### Example Workflow + +```yaml +name: Update Submodules to Latest Release + +on: + schedule: + - cron: '0 0 * * MON' # https://crontab.guru/every-monday + +jobs: + repo_submodule_update: + name: Check for Submodule Releases + runs-on: ubuntu-latest + + steps: + - name: Update Submodules to Latest Release + uses: microsoft/mu_devops/.github/actions/submodule-release-updater@v2.4.0 + with: + GH_PAT: ${{ secrets.SUBMODULE_UPDATER_TOKEN }} + GH_USER: "Add GitHub account username here" + GIT_EMAIL: "Add email address here" + GIT_NAME: "Add git author name here" + +``` + +## Action Inputs + +- `GH_PAT` - **Required** - GitHub Personal Access Token (PAT) with `repo` scope +- `GH_USER` - **Required** - GitHub username +- `GIT_EMAIL` - **Required** - Email address to use for git commits +- `GIT_NAME` - **Required** - Name to use for git commits + +## Action Outputs + +- `submodule-update-count` - Number of submodules updated. `0` if no submodules were updated. + +## Limitations + +- This action is only intended to work within Project Mu repositories. +- This action only supports repositories hosted on GitHub. +- This action only updates submodules that are hosted on GitHub. +- This action is only intended to work with submodules that use [semantic versioning](https://semver.org/). +- Submodules should already be set to a specific release before enabling this action. + - This allows the action to compare new versions to the current version. +- This action does not automatically close stale PRs when a new release is available. diff --git a/.github/actions/submodule-release-updater/action.yml b/.github/actions/submodule-release-updater/action.yml new file mode 100644 index 00000000..90df5ee7 --- /dev/null +++ b/.github/actions/submodule-release-updater/action.yml @@ -0,0 +1,425 @@ +# A GitHub action to create pull requests for new releases of submodules in a repository. +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +# + +name: 'Submodule Release Updater' + +description: 'Checks if new releases are available for submodules and creates pull requests to update them.' + +inputs: + GH_PAT: + description: 'GitHub Personal Access Token (PAT) used to access repos and create pull requests.' + required: true + GH_USER: + description: 'GitHub username used to create pull requests.' + required: true + GIT_EMAIL: + description: 'Email address used for authoring Git commits.' + required: true + GIT_NAME: + description: 'Name used for authoring Git commits.' + required: true + +outputs: + submodules-updated: + description: "Number of submodules updated." + value: ${{ steps.check-for-submodule-updates.outputs.submodule-update-count }} + +runs: + using: "composite" + + steps: + - name: Set up Python Environment + uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install PIP Modules + shell: bash + run: | + python -m pip install --upgrade pip + pip install GitPython requests semantic-version + + - name: Check for Submodule Updates + id: check-for-submodule-updates + shell: python + env: + GITHUB_TOKEN: "${{ inputs.GH_PAT }}" + GITHUB_USER: "${{ inputs.GH_USER }}" + GIT_EMAIL_ADDRESS: "${{ inputs.GIT_EMAIL }}" + GIT_NAME: "${{ inputs.GIT_NAME }}" + PR_LABELS: "${{ inputs.PR_LABELS }}" + run: | + import git + import json + import os + import re + import requests + import semantic_version + import sys + from textwrap import dedent + from urllib.parse import urlparse + + # Ignore flake8 linter errors for lines that are too long (E501) + # flake8: noqa: E501 + + AUTHORIZED_ORGANIZATIONS = "microsoft" # Assume "microsoft" org right now + GH_REPO = os.environ["GITHUB_REPOSITORY"] + GH_TOKEN = os.environ["GITHUB_TOKEN"] + GH_USER = os.environ["GITHUB_USER"] + GIT_EMAIL_ADDRESS = os.environ["GIT_EMAIL_ADDRESS"] + GIT_NAME = os.environ["GIT_NAME"] + PR_LABELS = ['type:dependencies', 'type:submodules'] + WORKSPACE_DIR_NAME = "local_clone" + WORKSPACE_PATH = os.environ["GITHUB_WORKSPACE"] + + + def _ver_without_prefix(version: str) -> str: + if len(version) == 0: + return "" + + ver_prefix = version.strip().lower()[0] + if ver_prefix == "v": + return version[len("v"):] + + return version + + + # GitHub REST API request and response documentation is available here: + # https://docs.github.com/en/rest?apiVersion=2022-11-28 + + remote = f"https://{GH_USER}:{GH_TOKEN}@github.com/{GH_REPO}.git" + repo_owner, repo_name = GH_REPO.split('/') + + headers = { + "Accept": "application/vnd.github.v3+json", + "X-GitHub-Api-Version": "2022-11-28" + } + submodule_headers = headers + headers["Authorization"] = f"Bearer {GH_TOKEN}" + + # Clone the repo using local creds + workspace_abs_path = os.path.join(WORKSPACE_PATH, WORKSPACE_DIR_NAME) + repo = git.Repo.clone_from(remote, workspace_abs_path) + repo.config_writer().set_value('user', 'name', GIT_NAME).release() + repo.config_writer().set_value('user', 'email', GIT_EMAIL_ADDRESS).release() + + base_branch = None + submodules = repo.submodules + + submodule_update_count = 0 + for submodule in submodules: + # The initial "querying" part of this flow relies upon the GitHub REST + # API which is must faster to query than initializing submodules + # locally + parsed_url = urlparse(submodule.url) + + # Only support GitHub repos for now + if "github" not in parsed_url.hostname: + print("::notice title=GitHub Host Not Found!::This workflow only " + "supports GitHub hosted repos!") + continue + + path = parsed_url.path.strip('/') + submod_user, submod_repo = os.path.split(path) + + submod_abs_path = os.path.join(workspace_abs_path, submodule.path) + submod_repo = submod_repo[:-len(".git")] if submod_repo.endswith(".git") else submod_repo + + authorized_orgs = AUTHORIZED_ORGANIZATIONS.split(',') + authorized_orgs = [s.strip() for s in authorized_orgs] + if any(org == submod_user for org in authorized_orgs): + # Use an auth token if possible to increase the access rate limit + submodule_headers["Authorization"] = f"Bearer {GH_TOKEN}" + + # Get the latest release for the submodule + response = requests.get( + f"https://api.github.com/repos/" + f"{submod_user}/{submod_repo}/releases/latest", + headers=submodule_headers) + if response.status_code == 200: + tag = response.json()["tag_name"] + else: + print(f"::notice title=Submodule Release Not Found!::Failed to " + f"query releases for {submod_repo}. Skipping!") + continue + + actual_available_tag = tag + available_tag = _ver_without_prefix(tag) + + print(f"::notice title=Available Submodule Tag Found!::Found {actual_available_tag} " + f"as the latest release tag for {submod_repo}.") + + response = requests.get( + f"https://api.github.com/repos/" + f"{submod_user}/{submod_repo}/git/refs/tags/{actual_available_tag}", + headers=submodule_headers) + if response.status_code != 200: + print(f"::error title=Commit For Release Tag Not Found!::Skipping " + f"submodule {submod_repo}.") + continue + + available_tag_commit_hash = response.json()["object"]["sha"] + + print(f"::notice title=New Release Commit Found!::Found " + f"{available_tag_commit_hash} as the commit for {actual_available_tag}.") + + try: + available_sem_ver = semantic_version.Version(available_tag) + print("::notice title=Semantic Version Tag!::The available tag is " + "recognized as a semantic version.") + except ValueError: + # Only semantic versioned tags are currently supported + print("::notice title=Non-Semantic Version Tag!::Skipping tag not " + "recognized as a semantic version.") + continue + + # Get the current submodule commit hash + response = requests.get( + f"https://api.github.com/repos/" + f"{repo_owner}/{repo_name}/contents/{submodule.path}", + headers=headers) + if response.status_code != 200: + print(f"::error title=Submodule Info Not Found!::Failed to find " + f"submodule info for {submod_repo}!") + continue + + current_tag_commit_hash = response.json()["sha"] + + print(f"::notice title=Current Submodule Commit Found!::Found " + f"{current_tag_commit_hash} as the commit for {submod_repo}.") + + # Get all of the submodule tags + response = requests.get( + f"https://api.github.com/repos/" + f"{submod_user}/{submod_repo}/git/refs/tags", + headers=submodule_headers) + if response.status_code != 200: + print(f"::error title=Failed to Get Submodule Tags!::Failed to get " + f"tags for {submod_repo}!") + continue + + # Find the most recent tag that contains the current commit hash + print(f"::notice title=Initializing Submodule...::Initializing " + f"{submod_repo}.") + submodule.update(init=True, recursive=False) + submodule_repo = git.Repo(submod_abs_path) + print(f"::notice title=Initialization Complete!::Done initializing " + f"{submod_repo}.") + + print(f"::notice title=Searching for Latest Tag Used!::Finding most " + f"recent tag used in {submod_repo}...") + actual_current_tag = None + actual_current_tag_committed_datetime = None + for tag in submodule_repo.tags: + tag_commit = submodule_repo.commit(tag.commit) + if current_tag_commit_hash in \ + [commit.hexsha for commit in tag_commit.iter_items(repo=submodule_repo, rev=tag.name)]: + # Find the "nearest" tag that contains the commit + if not actual_current_tag_committed_datetime or \ + (tag.commit.committed_datetime < actual_current_tag_committed_datetime): + actual_current_tag = tag.name + actual_current_tag_committed_datetime = tag.commit.committed_datetime + print("::notice title=Searching for Latest Tag Used!::Done!") + + if not actual_current_tag: + print(f"::notice title=Tag Not Found For Submodule Commit!::Could " + f"not find tag for commit {current_tag_commit_hash}. Skipping submodule.") + continue + + current_tag = _ver_without_prefix(actual_current_tag) + + print(f"::notice title=Current Submodule Tag Found!::{submod_repo} is " + f"currently on tag ({current_tag}).") + + try: + current_sem_ver = semantic_version.Version(current_tag) + print(f"::notice title=Semantic Version Tag!::{current_tag} is " + f"recognized as a semantic version.") + except ValueError: + # Only semantic versioned tags are currently supported + print(f"::notice title=Non-Semantic Version Tag!::Skipping tag " + f"({current_tag}) since it is not recognized as a semantic version.") + continue + + if available_sem_ver > current_sem_ver: + print(f"::notice title=Version Update Ready!::{submod_repo} can be " + f"updated from {current_tag} to {available_tag}.") + + response = requests.get( + f"https://api.github.com/repos/" + f"{submod_user}/{submod_repo}/compare/{actual_current_tag}...{actual_available_tag}", + headers=submodule_headers) + if response.status_code == 200: + print("::notice title=Commit Info Found!::Found commit delta " + "for the tag update.") + else: + # Commits should be available for existing tags + print("::error title=Commit Info Not Found!::Could not find " + "commit delta for the tag update}!") + continue + + tag_comp_response = response.json() + + if "commits" not in tag_comp_response: + # Not necessarily an error but no need to gather commit info + # if there are no commits + print(f"::notice title=Commits Not Found!::No new commits " + f"found in the new tag {actual_available_tag}!") + continue + + commit_summary = dedent(f""" + Introduces {tag_comp_response["total_commits"]} new commits in [{submodule.name}]({submodule.url}). + +
+ Commits +
    + """) + + for commit in tag_comp_response["commits"]: + commit_message = commit["commit"]["message"] + commit_title = commit_message.split("\n")[0] + + # Since the PR is in a different repo, replace a potential + # PR number in the commit title with an actual link to + # the PR in that repo. + pr_num_pattern = r"#(?P\d+)" + pr_url_template = f"\">#\\g" + + commit_title = re.sub( + pr_num_pattern, + pr_url_template, + commit_title) + + commit_summary += f"
  • {commit['sha'][:6]} {commit_title}
    • \n" + + commit_summary += dedent(""" +
+
+ """).strip() + + pr_body = dedent(f""" + Bumps {submodule.name} from `{current_tag}` to `{available_tag}` + + {commit_summary} + """).strip().strip("\n") + + pr_body += f"\n\nSigned-off-by: {GIT_NAME} <{GIT_EMAIL_ADDRESS}>" + + branch_name = f"projectmubot/submodules/{submod_repo}/{available_tag}" + + # Check if this update already exists on the remote + response = requests.get( + f"https://api.github.com/repos/" + f"{repo_owner}/{repo_name}/branches/{branch_name}", + headers=headers) + if response.status_code == 200: + print("::notice title=Update Already Exists!::This update " + "has already been pushed before. Skipping it.") + continue + + # Todo: Close PRs that already exist that update to an earlier + # version of a submodule release. + + # Get repo default branch + if not base_branch: + response = requests.get( + f"https://api.github.com/repos/" + f"{repo_owner}/{repo_name}", + headers=headers) + if response.status_code == 200: + base_branch = response.json()["default_branch"] + else: + # Commits should be available for existing tags + print(f"::error title=Default Branch Not Found!::Could " + f"not find the default branch for {repo_name}. Exiting.") + sys.exit(1) + + print(f"::notice title=Default Branch Found!::Default branch " + f"for {repo_name} is {base_branch}.") + + # Checkout the default branch + try: + repo.git.checkout(base_branch) + except git.exc.GitCommandError: + try: + repo.git.checkout('-b', base_branch) + except git.exc.GitCommandError: + print(f"::error title=Git Branch Checkout Failed!::" + f"Could not checkout {base_branch}. Exiting.") + sys.exit(1) + + # Create a local git branch from the default branch + try: + new_branch = repo.create_head(branch_name) + except OSError: + print(f"::error title=Failed to Create Branch!::Failed to " + f"create the branch needed to update PR. Skipping {submod_repo}.") + continue + repo.head.reference = new_branch + + # In the workflow, we assume the "origin" remote is available + origin = repo.remote(name="origin") + submodule_repo.remotes.origin.fetch() + + # Update the submodule to the release tag commit + # This has been shown to fail to apply on the first try so + # try up to 3 times + for i in range(3): + submodule_repo.git.reset('--hard', available_tag_commit_hash) + if submodule_repo.head.commit.hexsha == available_tag_commit_hash: + break + else: + print(f"::error title=Failed to Checkout New Commit!::Failed " + f"to checkout {available_tag_commit_hash}. Skipping.") + continue + + # Commit the change to the local branch + repo.git.add(submodule.path) + commit_message = pr_body + repo.index.commit(commit_message) + + # Push the branch + origin.push(new_branch) + + pr_payload = { + "title": f"Bump {submodule.name} from {current_tag} to {available_tag}", + "body": pr_body.replace("'", '"'), + "base": base_branch, + "head": branch_name, + } + + # Create the PR + response = requests.post( + f"https://api.github.com/repos/" + f"{repo_owner}/{repo_name}/pulls", + json=pr_payload, + headers=headers) + if response.status_code != 201: + print("::error title=Failed to Create PR!::Failed to " + "create the PR. Exiting.") + sys.exit(1) + + pr_number = response.json()["number"] + pr_url = response.json()["html_url"] + submodule_update_count += 1 + print(f"::notice title=PR Created!::{pr_url}") + + if PR_LABELS: + print(f"::notice title=Adding PR Labels::Adding labels to PR {pr_number}...") + + # Add labels to the PR + response = requests.post( + f"https://api.github.com/repos/" + f"{repo_owner}/{repo_name}/issues/{pr_number}/labels", + json=PR_LABELS, + headers=headers) + if response.status_code != 200: + print(f"::error title=Failed to Add Labels!::Could not " + f"add labels to PR {pr_number}.") + sys.exit(1) + + with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: + print(f'submodule-update-count={submodule_update_count}', file=fh) From 04d8a9a3894b49e7cae392ae4b6bb2d8c4cabd34 Mon Sep 17 00:00:00 2001 From: Michael Kubacki Date: Thu, 20 Apr 2023 15:32:15 -0400 Subject: [PATCH 2/3] .sync/Files.yml: Sync Submodule Release Update workflow Syncs a new workflow to update submodules to the latest GitHub release to mu_tiano_platforms. Signed-off-by: Michael Kubacki --- .sync/Files.yml | 8 +++++ .../leaf/submodule-release-update.yml | 35 +++++++++++++++++++ ReadMe.rst | 12 +++++++ 3 files changed, 55 insertions(+) create mode 100644 .sync/workflows/leaf/submodule-release-update.yml diff --git a/.sync/Files.yml b/.sync/Files.yml index 50640e51..3fffe137 100644 --- a/.sync/Files.yml +++ b/.sync/Files.yml @@ -538,6 +538,14 @@ group: microsoft/mu_tiano_platforms microsoft/mu_tiano_plus +# Leaf Workflow - Submodule Release Update + - files: + - source: .sync/workflows/leaf/submodule-release-update.yml + dest: .github/workflows/submodule-release-update.yml + template: true + repos: | + microsoft/mu_tiano_platforms + # Pull Request Template - Common Template - files: - source: .sync/github_templates/pull_requests/pull_request_template.md diff --git a/.sync/workflows/leaf/submodule-release-update.yml b/.sync/workflows/leaf/submodule-release-update.yml new file mode 100644 index 00000000..c00450ba --- /dev/null +++ b/.sync/workflows/leaf/submodule-release-update.yml @@ -0,0 +1,35 @@ +# This workflow automatically creates a pull request for any submodule in the repo +# that has a new GitHub release available. The release must follow semantic versioning. +# +# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there +# instead of the file in this repo. +# +# - Mu DevOps Repo: https://github.com/microsoft/mu_devops +# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +# + +{% import '../../Version.njk' as sync_version -%} + +name: Update Submodules to Latest Release + +on: + schedule: + - cron: '0 0 * * *' # https://crontab.guru/every-day + workflow_dispatch: + +jobs: + repo_submodule_update: + name: Check for Submodule Releases + runs-on: ubuntu-latest + + steps: + - name: Update Submodules to Latest Release + uses: microsoft/mu_devops/.github/actions/submodule-release-updater@{{ sync_version.mu_devops }} + with: + GH_PAT: ${{ secrets.SUBMODULE_UPDATER_TOKEN }} + GH_USER: "ProjectMuBot" + GIT_EMAIL: "mubot@microsoft.com" + GIT_NAME: "Project Mu Bot" diff --git a/ReadMe.rst b/ReadMe.rst index c87052b2..cfefe207 100644 --- a/ReadMe.rst +++ b/ReadMe.rst @@ -203,6 +203,18 @@ quality of pull request verbiage. - The leaf workflow - `.sync/workflows/leaf/pull-request-formatting-validator.yml` +Submodule Release Updater +------------------------- + +A GitHub Action and leaf workflow that automatically create a pull request for any submodule in a repo +that has a new GitHub release available. The leaf workflow can easily be synced to repos and wraps around +the GitHub action. + +- The GitHub action + - `.github/actions/submodule-release-updater` +- The leaf workflow + - `.sync/workflows/leaf/submodule-release-update.yml` + Links ===== - `Basic Azure Landing Site `_ From 5400508a9b6f19b5118f7f92973b6980d31f2073 Mon Sep 17 00:00:00 2001 From: Michael Kubacki Date: Thu, 20 Apr 2023 15:36:32 -0400 Subject: [PATCH 3/3] .sync/Version.njk: Update Mu repos to Mu DevOps v2.4.0 Changes since last release: https://github.com/microsoft/mu_devops/compare/v2.3.0...v2.4.0 General release info: https://github.com/microsoft/mu_devops/releases Signed-off-by: Michael Kubacki --- .sync/Version.njk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.sync/Version.njk b/.sync/Version.njk index f85e9714..8d95e27f 100644 --- a/.sync/Version.njk +++ b/.sync/Version.njk @@ -30,7 +30,7 @@ #} {# The git ref value that files dependent on this repo will use. #} -{% set mu_devops = "v2.3.0" %} +{% set mu_devops = "v2.4.0" %} {# The latest Project Mu release branch value. #} {% set latest_mu_release_branch = "release/202208" %}