Skip to content

Commit

Permalink
Merge Upstream Workflow (#1694)
Browse files Browse the repository at this point in the history
## Что этот PR делает
Добавляет воркфлоу, который мержит апстрим.
Чейнджлог собирается из обнаруженных в истории ПРов и переводится
(опционально).
Отслеживаются метки изменений в конфиге, бд, вики. ПРы с этими метками
отдельно упоминаются в теле ПРа.
Конфликты решать самому, увы.

Используется ветка для пула с апстрима, которая может быть создана, если
не существует.

В чейнджлоге остаются комментарии, содержащие текст исходного изменения,
если был перевод, и ссылку на оригинальный ПР.

```
fix: Сброс идентификатора агента больше не вызывает неизменного типа крови. <!-- fix: Resetting an agent ID no longer causes bloodtype to be unchangable. (ParadiseSS13#27509) -->
```

## Тестирование

Пример ПРа (автор я потому что токен был мой):
m-dzianishchyts#48

https://github.com/m-dzianishchyts/Paradise-SS220/actions/runs/12247152346/job/34164451774
  • Loading branch information
m-dzianishchyts authored Dec 10, 2024
1 parent 3443d5a commit b78dda5
Show file tree
Hide file tree
Showing 5 changed files with 457 additions and 115 deletions.
10 changes: 6 additions & 4 deletions .github/workflows/check_changelog.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ on:
jobs:
CheckCL:
runs-on: ubuntu-latest
if: github.repository == 'ss220club/Paradise-SS220' && github.base_ref == 'master' && github.event.pull_request.draft == false
if: github.base_ref == 'master' && github.event.pull_request.draft == false

steps:
- name: Downloading scripts
run: |
wget https://raw.githubusercontent.com/ss220club/Paradise-SS220/master/tools/changelog/check_changelog.py
wget https://raw.githubusercontent.com/ss220club/Paradise-SS220/master/tools/changelog/tags.yml
wget https://raw.githubusercontent.com/${{ github.repository }}/${{ github.base_ref }}/tools/changelog/changelog_utils.py
wget https://raw.githubusercontent.com/${{ github.repository }}/${{ github.base_ref }}/tools/changelog/check_changelog.py
wget https://raw.githubusercontent.com/${{ github.repository }}/${{ github.base_ref }}/tools/changelog/tags.yml
- name: Installing Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1
with:
Expand All @@ -28,5 +30,5 @@ jobs:
pip install ruamel.yaml PyGithub
- name: Changelog validation
env:
BOT_TOKEN: ${{ secrets.BOT_TOKEN }}
GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }}
run: python check_changelog.py
47 changes: 47 additions & 0 deletions .github/workflows/merge_upstream.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
name: Merge Upstream

on:
workflow_dispatch:

jobs:
merge-upstream:
runs-on: ubuntu-latest

steps:
- id: create_token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.PRIVATE_KEY }}

- run: echo "GH_TOKEN=${{ steps.create_token.outputs.token }}" >> "$GITHUB_ENV"

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.x

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install PyGithub googletrans==4.0.0-rc1
- name: Download the script
run: |
wget https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref_name }}/tools/changelog/changelog_utils.py
wget https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref_name }}/tools/merge-upstream/merge_upstream.py
- name: Run the script
env:
GITHUB_TOKEN: ${{ env.GH_TOKEN }}
TARGET_REPO: 'ss220club/Paradise-SS220'
TARGET_BRANCH: 'master'
UPSTREAM_REPO: 'ParadiseSS13/Paradise'
UPSTREAM_BRANCH: 'master'
MERGE_BRANCH: 'merge-upstream'
TRANSLATE_CHANGES: 'true'
CHANGELOG_AUTHOR: 'ParadiseSS13'
run: |
git config --global user.email "[email protected]"
git config --global user.name "Upstream Sync"
python3 merge_upstream.py
101 changes: 101 additions & 0 deletions tools/changelog/changelog_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
import re
import copy

CL_INVALID = ":scroll: CL невалиден"
CL_VALID = ":scroll: CL валиден"
CL_NOT_NEEDED = ":scroll: CL не требуется"

DISCORD_EMBED_DESCRIPTION_LIMIT = 4096

CL_BODY = re.compile(r"(:cl:|🆑)[ \t]*(?P<author>.+?)?\s*\n(?P<content>(.|\n)*?)\n/(:cl:|🆑)", re.MULTILINE)
CL_SPLIT = re.compile(r"\s*(?:(?P<tag>\w+)\s*:)?\s*(?P<message>.*)")

DISCORD_TAG_EMOJI = {
"soundadd": ":notes:",
"sounddel": ":mute:",
"imageadd": ":frame_photo:",
"imagedel": ":scissors:",
"codeadd": ":sparkles:",
"codedel": ":wastebasket:",
"tweak": ":screwdriver:",
"fix": ":tools:",
"wip": ":construction_site:",
"spellcheck": ":pencil:",
"experiment": ":microscope:"
}


def build_changelog(pr: dict, tags_config: dict) -> dict:
changelog = parse_changelog(pr.body, tags_config)
if changelog is None:
raise Exception("Failed to parse the changelog. Check changelog format.")
changelog["author"] = changelog["author"] or pr.user.login
return changelog


def emojify_changelog(changelog: dict):
changelog_copy = copy.deepcopy(changelog)
for change in changelog_copy["changes"]:
if change["tag"] in DISCORD_TAG_EMOJI:
change["tag"] = DISCORD_TAG_EMOJI[change["tag"]]
else:
raise Exception(f"Invalid tag for emoji: {change}")
return changelog_copy


def validate_changelog(changelog: dict):
if not changelog:
raise Exception("No changelog.")
if not changelog["author"]:
raise Exception("The changelog has no author.")
if len(changelog["changes"]) == 0:
raise Exception("No changes found in the changelog. Use special label if changelog is not expected.")
message = "\n".join(map(lambda change: f"{change['tag']} {change['message']}", changelog["changes"]))
if len(message) > DISCORD_EMBED_DESCRIPTION_LIMIT:
raise Exception(f"The changelog exceeds the length limit ({DISCORD_EMBED_DESCRIPTION_LIMIT}). Shorten it.")


def parse_changelog(pr_body: str, tags_config: dict | None = None) -> dict | None:
clean_pr_body = re.sub(r"<!--.*?-->", "", pr_body, flags=re.DOTALL)
cl_parse_result = CL_BODY.search(clean_pr_body)
if cl_parse_result is None:
return None

cl_changes = []
for cl_line in cl_parse_result.group("content").splitlines():
if not cl_line:
continue
change_parse_result = CL_SPLIT.search(cl_line)
if not change_parse_result:
raise Exception(f"Invalid change: '{cl_line}'")
tag = change_parse_result["tag"]
message = change_parse_result["message"]

if tags_config and tag and tag not in tags_config['tags'].keys():
raise Exception(f"Invalid tag: '{cl_line}'. Valid tags: {', '.join(tags_config['tags'].keys())}")
if not message:
raise Exception(f"No message for change: '{cl_line}'")

message = message.strip()

if tags_config and message in list(tags_config['defaults'].values()): # Check to see if the tags are associated with something that isn't the default text
raise Exception(f"Don't use default message for change: '{cl_line}'")
if tag:
cl_changes.append({
"tag": tags_config['tags'][tag] if tags_config else tag,
"message": message
})
# Append line without tag to the previous change
else:
if len(cl_changes):
prev_change = cl_changes[-1]
prev_change["message"] += f" {message}"
else:
raise Exception(f"Change with no tag: {cl_line}")

if len(cl_changes) == 0:
raise Exception("No changes found in the changelog. Use special label if changelog is not expected.")
return {
"author": str.strip(cl_parse_result.group("author") or "") or None, # I want this to be None, not empty
"changes": cl_changes
}
130 changes: 19 additions & 111 deletions tools/changelog/check_changelog.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,114 +2,23 @@
DO NOT MANUALLY RUN THIS SCRIPT.
---------------------------------
Expected envrionmental variables:
Expected environmental variables:
-----------------------------------
GITHUB_REPOSITORY: Github action variable representing the active repo (Action provided)
BOT_TOKEN: A repository account token, this will allow the action to push the changes (Action provided)
GITHUB_TOKEN: A repository account token, this will allow the action to push the changes (Action provided)
GITHUB_EVENT_PATH: path to JSON file containing the event info (Action provided)
"""
import os
import re
import copy
from pathlib import Path
from ruamel.yaml import YAML
from github import Github
import json

DISCORD_EMBED_DESCRIPTION_LIMIT = 4096

CL_BODY = re.compile(r"(:cl:|🆑)[ \t]*(?P<author>.+?)?\s*\n(?P<content>(.|\n)*?)\n/(:cl:|🆑)", re.MULTILINE)
CL_SPLIT = re.compile(r"\s*(?:<!--.*-->)?((?P<tag>\w+)\s*:)?\s*(?P<message>.*)")

DISCORD_TAG_EMOJI = {
"soundadd": ":notes:",
"sounddel": ":mute:",
"imageadd": ":frame_photo:",
"imagedel": ":scissors:",
"codeadd": ":sparkles:",
"codedel": ":wastebasket:",
"tweak": ":screwdriver:",
"fix": ":tools:",
"wip": ":construction_site:",
"spellcheck": ":pencil:",
"experiment": ":microscope:"
}


def build_changelog(pr: dict) -> dict:
changelog = parse_changelog(pr.body)
changelog["author"] = changelog["author"] or pr.user.login
return changelog


def emojify_changelog(changelog: dict):
changelog_copy = copy.deepcopy(changelog)
for change in changelog_copy["changes"]:
if change["tag"] in DISCORD_TAG_EMOJI:
change["tag"] = DISCORD_TAG_EMOJI[change["tag"]]
else:
raise Exception(f"Invalid tag for emoji: {change}")
return changelog_copy


def validate_changelog(changelog: dict):
if not changelog:
raise Exception("No changelog.")
if not changelog["author"]:
raise Exception("The changelog has no author.")
if len(changelog["changes"]) == 0:
raise Exception("No changes found in the changelog. Use special label if changelog is not expected.")
message = "\n".join(map(lambda change: f"{change['tag']} {change['message']}", changelog["changes"]))
if len(message) > DISCORD_EMBED_DESCRIPTION_LIMIT:
raise Exception(f"The changelog exceeds the length limit ({DISCORD_EMBED_DESCRIPTION_LIMIT}). Shorten it.")


def parse_changelog(message: str) -> dict:
with open(Path.cwd().joinpath("tags.yml")) as file:
yaml = YAML(typ = 'safe', pure = True)
tags_config = yaml.load(file)
cl_parse_result = CL_BODY.search(message)
if cl_parse_result is None:
raise Exception("Failed to parse the changelog. Check changelog format.")
cl_changes = []
for cl_line in cl_parse_result.group("content").splitlines():
if not cl_line:
continue
change_parse_result = CL_SPLIT.search(cl_line)
if not change_parse_result:
raise Exception(f"Invalid change: '{cl_line}'")
tag = change_parse_result["tag"]
message = change_parse_result["message"]
if tag and tag not in tags_config['tags'].keys():
raise Exception(f"Invalid tag: '{cl_line}'. Valid tags: {', '.join(tags_config['tags'].keys())}")
if not message:
raise Exception(f"No message for change: '{cl_line}'")
if message in list(tags_config['defaults'].values()): # Check to see if the tags are associated with something that isn't the default text
raise Exception(f"Don't use default message for change: '{cl_line}'")
if tag:
cl_changes.append({
"tag": tags_config['tags'][tag],
"message": message
})
# Append line without tag to the previous change
else:
if len(cl_changes):
prev_change = cl_changes[-1]
prev_change["message"] += f" {message}"
else:
raise Exception(f"Change with no tag: {cl_line}")

if len(cl_changes) == 0:
raise Exception("No changes found in the changelog. Use special label if changelog is not expected.")
return {
"author": str.strip(cl_parse_result.group("author") or "") or None, # I want this to be None, not empty
"changes": cl_changes
}

import changelog_utils

# Blessed is the GoOnStAtIoN birb ZeWaKa for thinking of this first
repo = os.getenv("GITHUB_REPOSITORY")
token = os.getenv("BOT_TOKEN")
token = os.getenv("GITHUB_TOKEN")
event_path = os.getenv("GITHUB_EVENT_PATH")

with open(event_path, 'r') as f:
Expand All @@ -123,23 +32,19 @@ def parse_changelog(message: str) -> dict:
pr_author = pr.user.login
pr_labels = pr.labels

CL_INVALID = ":scroll: CL невалиден"
CL_VALID = ":scroll: CL валиден"
CL_NOT_NEEDED = ":scroll: CL не требуется"

pr_is_mirror = pr.title.startswith("[MIRROR]")

has_valid_label = False
has_invalid_label = False
cl_required = True
for label in pr_labels:
print("Found label: ", label.name)
if label.name == CL_NOT_NEEDED:
if label.name == changelog_utils.CL_NOT_NEEDED:
print("No CL needed!")
cl_required = False
if label.name == CL_VALID:
if label.name == changelog_utils.CL_VALID:
has_valid_label = True
if label.name == CL_INVALID:
if label.name == changelog_utils.CL_INVALID:
has_invalid_label = True

if pr_is_mirror:
Expand All @@ -148,30 +53,33 @@ def parse_changelog(message: str) -> dict:
if not cl_required:
# remove invalid, remove valid
if has_invalid_label:
pr.remove_from_labels(CL_INVALID)
pr.remove_from_labels(changelog_utils.CL_INVALID)
if has_valid_label:
pr.remove_from_labels(CL_VALID)
pr.remove_from_labels(changelog_utils.CL_VALID)
exit(0)

try:
cl = build_changelog(pr)
cl_emoji = emojify_changelog(cl)
with open(Path.cwd().joinpath("tags.yml")) as file:
yaml = YAML(typ = 'safe', pure = True)
tags_config = yaml.load(file)
cl = changelog_utils.build_changelog(pr, tags_config)
cl_emoji = changelog_utils.emojify_changelog(cl)
cl_emoji["author"] = cl_emoji["author"] or pr_author
validate_changelog(cl_emoji)
changelog_utils.validate_changelog(cl_emoji)
except Exception as e:
print("Changelog parsing error:")
print(e)

# add invalid, remove valid
if not has_invalid_label:
pr.add_to_labels(CL_INVALID)
pr.add_to_labels(changelog_utils.CL_INVALID)
if has_valid_label:
pr.remove_from_labels(CL_VALID)
pr.remove_from_labels(changelog_utils.CL_VALID)
exit(1)

# remove invalid, add valid
if has_invalid_label:
pr.remove_from_labels(CL_INVALID)
pr.remove_from_labels(changelog_utils.CL_INVALID)
if not has_valid_label:
pr.add_to_labels(CL_VALID)
pr.add_to_labels(changelog_utils.CL_VALID)
print("Changelog is valid.")
Loading

0 comments on commit b78dda5

Please sign in to comment.