From 9917b21a6e3c1d28eb4bab4d0184ab8a72360373 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Sun, 8 Dec 2024 17:20:54 +0300 Subject: [PATCH 1/9] Add `merge-upstream` workflow --- .github/workflows/merge_upstream.yml | 36 +++++ tools/changelog/changelog_utils.py | 93 ++++++++++++ tools/changelog/check_changelog.py | 124 +++------------- tools/merge-upstream/merge_upstream.py | 189 +++++++++++++++++++++++++ 4 files changed, 334 insertions(+), 108 deletions(-) create mode 100644 .github/workflows/merge_upstream.yml create mode 100644 tools/changelog/changelog_utils.py create mode 100644 tools/merge-upstream/merge_upstream.py diff --git a/.github/workflows/merge_upstream.yml b/.github/workflows/merge_upstream.yml new file mode 100644 index 000000000000..b4b531d8af37 --- /dev/null +++ b/.github/workflows/merge_upstream.yml @@ -0,0 +1,36 @@ +name: Merge Upstream + +on: + workflow_dispatch: + +jobs: + merge-upstream: + runs-on: ubuntu-latest + + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.x + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install github googletrans + + - name: Download the script + run: | + wget https://raw.githubusercontent.com/ss220club/Paradise-SS220/master/tools/changelog/changelog_utils.py + wget https://raw.githubusercontent.com/ss220club/Paradise-SS220/master/tools/merge-upstream/merge_upstream.py + + - name: Run the changelog sync script + env: + GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }} + TARGET_REPO: "ss220club/Paradise-SS220" + TARGET_BRANCH: "master" + UPSTREAM_REPO: "ParadiseSS13/Paradise" + UPSTREAM_BRANCH: "master" + MERGE_BRANCH: "merge-upstream" + TRANSLATE_CHANGES: "true" + run: | + python3 merge_upstream.py diff --git a/tools/changelog/changelog_utils.py b/tools/changelog/changelog_utils.py new file mode 100644 index 000000000000..4243a752564e --- /dev/null +++ b/tools/changelog/changelog_utils.py @@ -0,0 +1,93 @@ +import re +import copy + +CL_INVALID = ":scroll: CL невалиден" +CL_VALID = ":scroll: CL валиден" +CL_NOT_NEEDED = ":scroll: CL не требуется" + +DISCORD_EMBED_DESCRIPTION_LIMIT = 4096 + +CL_BODY = re.compile(r"(:cl:|🆑)[ \t]*(?P.+?)?\s*\n(?P(.|\n)*?)\n/(:cl:|🆑)", re.MULTILINE) +CL_SPLIT = re.compile(r"\s*(?:)?((?P\w+)\s*:)?\s*(?P.*)") + +DISCORD_TAG_EMOJI = { + "soundadd": ":notes:", + "sounddel": ":mute:", + "imageadd": ":frame_photo:", + "imagedel": ":scissors:", + "codeadd": ":sparkles:", + "codedel": ":wastebasket:", + "tweak": ":screwdriver:", + "fix": ":tools:", + "wip": ":construction_site:", + "spellcheck": ":pencil:", + "experiment": ":microscope:" +} + + +def build_changelog(pr: dict, tags_config: dict) -> dict: + changelog = parse_changelog(pr.body, tags_config) + changelog["author"] = changelog["author"] or pr.user.login + return changelog + + +def emojify_changelog(changelog: dict): + changelog_copy = copy.deepcopy(changelog) + for change in changelog_copy["changes"]: + if change["tag"] in DISCORD_TAG_EMOJI: + change["tag"] = DISCORD_TAG_EMOJI[change["tag"]] + else: + raise Exception(f"Invalid tag for emoji: {change}") + return changelog_copy + + +def validate_changelog(changelog: dict): + if not changelog: + raise Exception("No changelog.") + if not changelog["author"]: + raise Exception("The changelog has no author.") + if len(changelog["changes"]) == 0: + raise Exception("No changes found in the changelog. Use special label if changelog is not expected.") + message = "\n".join(map(lambda change: f"{change['tag']} {change['message']}", changelog["changes"])) + if len(message) > DISCORD_EMBED_DESCRIPTION_LIMIT: + raise Exception(f"The changelog exceeds the length limit ({DISCORD_EMBED_DESCRIPTION_LIMIT}). Shorten it.") + + +def parse_changelog(message: str, tags_config: dict) -> dict: + cl_parse_result = CL_BODY.search(message) + if cl_parse_result is None: + raise Exception("Failed to parse the changelog. Check changelog format.") + cl_changes = [] + for cl_line in cl_parse_result.group("content").splitlines(): + if not cl_line: + continue + change_parse_result = CL_SPLIT.search(cl_line) + if not change_parse_result: + raise Exception(f"Invalid change: '{cl_line}'") + tag = change_parse_result["tag"] + message = change_parse_result["message"] + if tag and tag not in tags_config['tags'].keys(): + raise Exception(f"Invalid tag: '{cl_line}'. Valid tags: {', '.join(tags_config['tags'].keys())}") + if not message: + raise Exception(f"No message for change: '{cl_line}'") + if message in list(tags_config['defaults'].values()): # Check to see if the tags are associated with something that isn't the default text + raise Exception(f"Don't use default message for change: '{cl_line}'") + if tag: + cl_changes.append({ + "tag": tags_config['tags'][tag], + "message": message + }) + # Append line without tag to the previous change + else: + if len(cl_changes): + prev_change = cl_changes[-1] + prev_change["message"] += f" {message}" + else: + raise Exception(f"Change with no tag: {cl_line}") + + if len(cl_changes) == 0: + raise Exception("No changes found in the changelog. Use special label if changelog is not expected.") + return { + "author": str.strip(cl_parse_result.group("author") or "") or None, # I want this to be None, not empty + "changes": cl_changes + } diff --git a/tools/changelog/check_changelog.py b/tools/changelog/check_changelog.py index 4b72cd0bc0b1..944bae6a57a3 100644 --- a/tools/changelog/check_changelog.py +++ b/tools/changelog/check_changelog.py @@ -9,103 +9,12 @@ GITHUB_EVENT_PATH: path to JSON file containing the event info (Action provided) """ import os -import re -import copy from pathlib import Path from ruamel.yaml import YAML from github import Github import json -DISCORD_EMBED_DESCRIPTION_LIMIT = 4096 - -CL_BODY = re.compile(r"(:cl:|🆑)[ \t]*(?P.+?)?\s*\n(?P(.|\n)*?)\n/(:cl:|🆑)", re.MULTILINE) -CL_SPLIT = re.compile(r"\s*((?P\w+)\s*:)?\s*(?P.*)") - -DISCORD_TAG_EMOJI = { - "soundadd": ":notes:", - "sounddel": ":mute:", - "imageadd": ":frame_photo:", - "imagedel": ":scissors:", - "codeadd": ":sparkles:", - "codedel": ":wastebasket:", - "tweak": ":screwdriver:", - "fix": ":tools:", - "wip": ":construction_site:", - "spellcheck": ":pencil:", - "experiment": ":microscope:" -} - - -def build_changelog(pr: dict) -> dict: - changelog = parse_changelog(pr.body) - changelog["author"] = changelog["author"] or pr.user.login - return changelog - - -def emojify_changelog(changelog: dict): - changelog_copy = copy.deepcopy(changelog) - for change in changelog_copy["changes"]: - if change["tag"] in DISCORD_TAG_EMOJI: - change["tag"] = DISCORD_TAG_EMOJI[change["tag"]] - else: - raise Exception(f"Invalid tag for emoji: {change}") - return changelog_copy - - -def validate_changelog(changelog: dict): - if not changelog: - raise Exception("No changelog.") - if not changelog["author"]: - raise Exception("The changelog has no author.") - if len(changelog["changes"]) == 0: - raise Exception("No changes found in the changelog. Use special label if changelog is not expected.") - message = "\n".join(map(lambda change: f"{change['tag']} {change['message']}", changelog["changes"])) - if len(message) > DISCORD_EMBED_DESCRIPTION_LIMIT: - raise Exception(f"The changelog exceeds the length limit ({DISCORD_EMBED_DESCRIPTION_LIMIT}). Shorten it.") - - -def parse_changelog(message: str) -> dict: - with open(Path.cwd().joinpath("tags.yml")) as file: - yaml = YAML(typ = 'safe', pure = True) - tags_config = yaml.load(file) - cl_parse_result = CL_BODY.search(message) - if cl_parse_result is None: - raise Exception("Failed to parse the changelog. Check changelog format.") - cl_changes = [] - for cl_line in cl_parse_result.group("content").splitlines(): - if not cl_line: - continue - change_parse_result = CL_SPLIT.search(cl_line) - if not change_parse_result: - raise Exception(f"Invalid change: '{cl_line}'") - tag = change_parse_result["tag"] - message = change_parse_result["message"] - if tag and tag not in tags_config['tags'].keys(): - raise Exception(f"Invalid tag: '{cl_line}'. Valid tags: {', '.join(tags_config['tags'].keys())}") - if not message: - raise Exception(f"No message for change: '{cl_line}'") - if message in list(tags_config['defaults'].values()): # Check to see if the tags are associated with something that isn't the default text - raise Exception(f"Don't use default message for change: '{cl_line}'") - if tag: - cl_changes.append({ - "tag": tags_config['tags'][tag], - "message": message - }) - # Append line without tag to the previous change - else: - if len(cl_changes): - prev_change = cl_changes[-1] - prev_change["message"] += f" {change_parse_result['message']}" - else: - raise Exception(f"Change with no tag: {cl_line}") - - if len(cl_changes) == 0: - raise Exception("No changes found in the changelog. Use special label if changelog is not expected.") - return { - "author": str.strip(cl_parse_result.group("author") or "") or None, # I want this to be None, not empty - "changes": cl_changes - } - +import changelog_utils # Blessed is the GoOnStAtIoN birb ZeWaKa for thinking of this first repo = os.getenv("GITHUB_REPOSITORY") @@ -123,10 +32,6 @@ def parse_changelog(message: str) -> dict: pr_author = pr.user.login pr_labels = pr.labels -CL_INVALID = ":scroll: CL невалиден" -CL_VALID = ":scroll: CL валиден" -CL_NOT_NEEDED = ":scroll: CL не требуется" - pr_is_mirror = pr.title.startswith("[MIRROR]") has_valid_label = False @@ -134,12 +39,12 @@ def parse_changelog(message: str) -> dict: cl_required = True for label in pr_labels: print("Found label: ", label.name) - if label.name == CL_NOT_NEEDED: + if label.name == changelog_utils.CL_NOT_NEEDED: print("No CL needed!") cl_required = False - if label.name == CL_VALID: + if label.name == changelog_utils.CL_VALID: has_valid_label = True - if label.name == CL_INVALID: + if label.name == changelog_utils.CL_INVALID: has_invalid_label = True if pr_is_mirror: @@ -148,30 +53,33 @@ def parse_changelog(message: str) -> dict: if not cl_required: # remove invalid, remove valid if has_invalid_label: - pr.remove_from_labels(CL_INVALID) + pr.remove_from_labels(changelog_utils.CL_INVALID) if has_valid_label: - pr.remove_from_labels(CL_VALID) + pr.remove_from_labels(changelog_utils.CL_VALID) exit(0) try: - cl = build_changelog(pr) - cl_emoji = emojify_changelog(cl) + with open(Path.cwd().joinpath("tags.yml")) as file: + yaml = YAML(typ = 'safe', pure = True) + tags_config = yaml.load(file) + cl = changelog_utils.build_changelog(pr, tags_config) + cl_emoji = changelog_utils.emojify_changelog(cl) cl_emoji["author"] = cl_emoji["author"] or pr_author - validate_changelog(cl_emoji) + changelog_utils.validate_changelog(cl_emoji) except Exception as e: print("Changelog parsing error:") print(e) # add invalid, remove valid if not has_invalid_label: - pr.add_to_labels(CL_INVALID) + pr.add_to_labels(changelog_utils.CL_INVALID) if has_valid_label: - pr.remove_from_labels(CL_VALID) + pr.remove_from_labels(changelog_utils.CL_VALID) exit(1) # remove invalid, add valid if has_invalid_label: - pr.remove_from_labels(CL_INVALID) + pr.remove_from_labels(changelog_utils.CL_INVALID) if not has_valid_label: - pr.add_to_labels(CL_VALID) + pr.add_to_labels(changelog_utils.CL_VALID) print("Changelog is valid.") diff --git a/tools/merge-upstream/merge_upstream.py b/tools/merge-upstream/merge_upstream.py new file mode 100644 index 000000000000..788098fbb0ca --- /dev/null +++ b/tools/merge-upstream/merge_upstream.py @@ -0,0 +1,189 @@ +import os +import re +import subprocess +import time +from datetime import datetime +from concurrent.futures import ThreadPoolExecutor, as_completed +from github import Github +from googletrans import Translator + +import changelog_utils + + +def check_env(): + """Check if the required environment variables are set.""" + required_vars = [ + "GITHUB_TOKEN", + "TARGET_REPO", + "TARGET_BRANCH", + "UPSTREAM_REPO", + "UPSTREAM_BRANCH", + "MERGE_BRANCH" + ] + missing_vars = [var for var in required_vars if not os.getenv(var)] + if missing_vars: + raise EnvironmentError(f"Missing required environment variables: {', '.join(missing_vars)}") + + +# Environment variables +check_env() +GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") +TARGET_REPO = os.getenv("TARGET_REPO") +TARGET_BRANCH = os.getenv("TARGET_BRANCH") +UPSTREAM_REPO = os.getenv("UPSTREAM_REPO") +UPSTREAM_BRANCH = os.getenv("UPSTREAM_BRANCH") +MERGE_BRANCH = os.getenv("MERGE_BRANCH") +TRANSLATE_CHANGES = os.getenv("TRANSLATE_CHANGES", "False").lower() in ("true", "yes", "1") + + +def run_command(command) -> str: + """Run a shell command and return its output.""" + result = subprocess.run(command, shell=True, capture_output=True, text=True) + result.check_returncode() + return result.stdout.strip() + + +def setup_repo(): + """Clone the repository and set up the upstream remote.""" + print(f"Cloning repository: {TARGET_REPO}") + run_command(f"git clone https://github.com/{TARGET_REPO}.git repo") + os.chdir("repo") + run_command(f"git remote add upstream https://github.com/{UPSTREAM_REPO}.git") + + +def update_merge_branch() -> bool: + """Update the merge branch with the latest changes from upstream.""" + print(f"Fetching branch {UPSTREAM_BRANCH} from upstream...") + run_command(f"git fetch upstream {UPSTREAM_BRANCH}") + + local_branches = run_command("git branch --list").split() + + if MERGE_BRANCH not in local_branches: + print(f"Branch '{MERGE_BRANCH}' does not exist. Creating it from upstream/{UPSTREAM_BRANCH}...") + run_command(f"git checkout -b {MERGE_BRANCH} upstream/{UPSTREAM_BRANCH}") + run_command(f"git push -u origin {MERGE_BRANCH}") + return True + + else: + print(f"Rebasing {MERGE_BRANCH} onto upstream/{UPSTREAM_BRANCH}...") + run_command(f"git checkout {MERGE_BRANCH}") + result = run_command(f"git pull --rebase upstream {UPSTREAM_BRANCH}") + + if "Current branch is up to date" in result: + print("No changes detected from upstream.") + return False + + print("Pushing rebased changes to origin...") + run_command(f"git push origin {MERGE_BRANCH} --force") + return True + + +def detect_commits() -> list[str]: + """Detect commits from upstream not yet in downstream.""" + print("Detecting new commits from upstream...") + return run_command(f"git log {TARGET_BRANCH}..{MERGE_BRANCH} --pretty=format:'%h %s %b'").split("\n") + + +def fetch_pull_body(pull_id) -> str | None: + """Fetch the pull request body from GitHub.""" + github = Github(GITHUB_TOKEN) + repo = github.get_repo(UPSTREAM_REPO) + + max_retries = 3 + for attempt in range(max_retries): + try: + pr = repo.get_pull(int(pull_id)) + return pr.body + except Exception as e: + print(f"Error fetching PR #{pull_id}: {e}") + if attempt + 1 < max_retries: + time.sleep(2) + else: + return None + + +def build_changelog(commit_log: list[str]) -> dict: + """Generate the changelog from parsed commits.""" + translator = Translator() + changelog = {} + + with ThreadPoolExecutor() as executor: + futures = {} + for commit in commit_log: + pull = re.search("#\d+", commit).group() + if not pull: + continue + + pull_id = pull[1:] + futures[executor.submit(fetch_pull_body, pull_id)] = pull_id + + for future in as_completed(futures): + pull_id = futures[future] + pull_body = future.result() + pull_url = f"https://github.com/{UPSTREAM_REPO}/pull/{pull_id}" + pull_changes = [] + + if not pull_body: + continue + + parsed = changelog_utils.parse_changelog(pull_body) + if parsed and parsed["changes"]: + for change in parsed["changes"]: + tag = change["tag"] + message = change["message"] + if TRANSLATE_CHANGES: + translated_message = translator.translate(message, src="en", dest="ru").text + pull_changes.append(f"{tag}: {translated_message}") + else: + pull_changes.append(f"{tag}: {message}") + pull_changes.append(f"") + + if pull_changes: + changelog[pull] = pull_changes + + return changelog + + +def prepare_pull_body(changelog: dict) -> str: + """Build new pull request body from the generated changelog.""" + pull_body = ( + f"This pull request merges upstream/{UPSTREAM_BRANCH}. " + f"Resolve possible conflicts manually and make sure all the changes applies correctly.\n" + ) + + if not changelog or not changelog.items(): + return pull_body + + pull_body += ( + f"\n## Changelog\n" + f":cl:\n" + ) + for change in changelog.values(): + pull_body += f"{change}\n" + pull_body += f"/:cl:\n" + + return pull_body + + +def create_pr(pull_body: str): + """Create a pull request with the processed changelog.""" + github = Github(GITHUB_TOKEN) + repo = github.get_repo(TARGET_REPO) + + # Create the pull request + repo.create_pull( + title=f"Merge Upstream {datetime.today().strftime('%d.%m.%Y')}", + body=pull_body, + head=TARGET_BRANCH, + base=MERGE_BRANCH + ) + + +if __name__ == "__main__": + setup_repo() + + if update_merge_branch(): + commit_log = detect_commits() + changelog = build_changelog(commit_log) + pull_body = prepare_pull_body(changelog) + create_pr(pull_body) From d02b0dbc99ccbefb5517d330638df98a1990386e Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Sun, 8 Dec 2024 17:40:46 +0300 Subject: [PATCH 2/9] Apply workflow fixes --- .github/workflows/check_changelog.yml | 2 +- .github/workflows/merge_upstream.yml | 23 +++-- tools/changelog/changelog_utils.py | 13 ++- tools/merge-upstream/merge_upstream.py | 132 +++++++++++++++---------- 4 files changed, 104 insertions(+), 66 deletions(-) diff --git a/.github/workflows/check_changelog.yml b/.github/workflows/check_changelog.yml index 25d12ce2f5fe..ddff67582a83 100644 --- a/.github/workflows/check_changelog.yml +++ b/.github/workflows/check_changelog.yml @@ -28,5 +28,5 @@ jobs: pip install ruamel.yaml PyGithub - name: Changelog validation env: - BOT_TOKEN: ${{ secrets.BOT_TOKEN }} + GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }} run: python check_changelog.py diff --git a/.github/workflows/merge_upstream.yml b/.github/workflows/merge_upstream.yml index b4b531d8af37..339bc405deab 100644 --- a/.github/workflows/merge_upstream.yml +++ b/.github/workflows/merge_upstream.yml @@ -16,21 +16,24 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install github googletrans + pip install PyGithub googletrans==4.0.0-rc1 - name: Download the script run: | - wget https://raw.githubusercontent.com/ss220club/Paradise-SS220/master/tools/changelog/changelog_utils.py - wget https://raw.githubusercontent.com/ss220club/Paradise-SS220/master/tools/merge-upstream/merge_upstream.py + wget https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref_name }}/tools/changelog/changelog_utils.py + wget https://raw.githubusercontent.com/${{ github.repository }}/${{ github.ref_name }}/tools/merge-upstream/merge_upstream.py - - name: Run the changelog sync script + - name: Run the script env: GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }} - TARGET_REPO: "ss220club/Paradise-SS220" - TARGET_BRANCH: "master" - UPSTREAM_REPO: "ParadiseSS13/Paradise" - UPSTREAM_BRANCH: "master" - MERGE_BRANCH: "merge-upstream" - TRANSLATE_CHANGES: "true" + TARGET_REPO: 'ss220club/Paradise-SS220' + TARGET_BRANCH: 'master' + UPSTREAM_REPO: 'ParadiseSS13/Paradise' + UPSTREAM_BRANCH: 'master' + MERGE_BRANCH: 'merge-upstream' + TRANSLATE_CHANGES: 'true' + CHANGELOG_AUTHOR: 'ParadiseSS13' run: | + git config --global user.email "action@github.com" + git config --global user.name "Upstream Sync" python3 merge_upstream.py diff --git a/tools/changelog/changelog_utils.py b/tools/changelog/changelog_utils.py index 4243a752564e..2f73343d872f 100644 --- a/tools/changelog/changelog_utils.py +++ b/tools/changelog/changelog_utils.py @@ -27,6 +27,8 @@ def build_changelog(pr: dict, tags_config: dict) -> dict: changelog = parse_changelog(pr.body, tags_config) + if changelog is None: + raise Exception("Failed to parse the changelog. Check changelog format.") changelog["author"] = changelog["author"] or pr.user.login return changelog @@ -53,10 +55,11 @@ def validate_changelog(changelog: dict): raise Exception(f"The changelog exceeds the length limit ({DISCORD_EMBED_DESCRIPTION_LIMIT}). Shorten it.") -def parse_changelog(message: str, tags_config: dict) -> dict: +def parse_changelog(message: str, tags_config: dict = {}) -> dict | None: cl_parse_result = CL_BODY.search(message) if cl_parse_result is None: - raise Exception("Failed to parse the changelog. Check changelog format.") + return None + cl_changes = [] for cl_line in cl_parse_result.group("content").splitlines(): if not cl_line: @@ -66,15 +69,15 @@ def parse_changelog(message: str, tags_config: dict) -> dict: raise Exception(f"Invalid change: '{cl_line}'") tag = change_parse_result["tag"] message = change_parse_result["message"] - if tag and tag not in tags_config['tags'].keys(): + if tags_config and tag and tag not in tags_config['tags'].keys(): raise Exception(f"Invalid tag: '{cl_line}'. Valid tags: {', '.join(tags_config['tags'].keys())}") if not message: raise Exception(f"No message for change: '{cl_line}'") - if message in list(tags_config['defaults'].values()): # Check to see if the tags are associated with something that isn't the default text + if tags_config and message in list(tags_config['defaults'].values()): # Check to see if the tags are associated with something that isn't the default text raise Exception(f"Don't use default message for change: '{cl_line}'") if tag: cl_changes.append({ - "tag": tags_config['tags'][tag], + "tag": tags_config['tags'][tag] if tags_config else tag, "message": message }) # Append line without tag to the previous change diff --git a/tools/merge-upstream/merge_upstream.py b/tools/merge-upstream/merge_upstream.py index 788098fbb0ca..801248706c4d 100644 --- a/tools/merge-upstream/merge_upstream.py +++ b/tools/merge-upstream/merge_upstream.py @@ -34,54 +34,60 @@ def check_env(): UPSTREAM_BRANCH = os.getenv("UPSTREAM_BRANCH") MERGE_BRANCH = os.getenv("MERGE_BRANCH") TRANSLATE_CHANGES = os.getenv("TRANSLATE_CHANGES", "False").lower() in ("true", "yes", "1") +CHANGELOG_AUTHOR = os.getenv("CHANGELOG_AUTHOR", "") def run_command(command) -> str: """Run a shell command and return its output.""" - result = subprocess.run(command, shell=True, capture_output=True, text=True) - result.check_returncode() - return result.stdout.strip() + try: + result = subprocess.run(command, shell=True, capture_output=True, text=True) + result.check_returncode() + return result.stdout.strip() + except subprocess.CalledProcessError as e: + print(f"Error executing command: {command}\nExit code: {e.returncode}\nOutput: {e.output}\nError: {e.stderr}") + raise def setup_repo(): """Clone the repository and set up the upstream remote.""" print(f"Cloning repository: {TARGET_REPO}") - run_command(f"git clone https://github.com/{TARGET_REPO}.git repo") + run_command(f"git clone https://{GITHUB_TOKEN}@github.com/{TARGET_REPO}.git repo") os.chdir("repo") - run_command(f"git remote add upstream https://github.com/{UPSTREAM_REPO}.git") + run_command(f"git remote add upstream https://{GITHUB_TOKEN}@github.com/{UPSTREAM_REPO}.git") + print(run_command(f"git remote -v")) -def update_merge_branch() -> bool: +def update_merge_branch(): """Update the merge branch with the latest changes from upstream.""" print(f"Fetching branch {UPSTREAM_BRANCH} from upstream...") run_command(f"git fetch upstream {UPSTREAM_BRANCH}") + run_command(f"git fetch origin") + all_branches = run_command("git branch -a").split() - local_branches = run_command("git branch --list").split() - - if MERGE_BRANCH not in local_branches: + if f"remotes/origin/{MERGE_BRANCH}" not in all_branches: print(f"Branch '{MERGE_BRANCH}' does not exist. Creating it from upstream/{UPSTREAM_BRANCH}...") run_command(f"git checkout -b {MERGE_BRANCH} upstream/{UPSTREAM_BRANCH}") run_command(f"git push -u origin {MERGE_BRANCH}") - return True + return - else: - print(f"Rebasing {MERGE_BRANCH} onto upstream/{UPSTREAM_BRANCH}...") - run_command(f"git checkout {MERGE_BRANCH}") - result = run_command(f"git pull --rebase upstream {UPSTREAM_BRANCH}") + print(f"Rebasing {MERGE_BRANCH} onto upstream/{UPSTREAM_BRANCH}...") + run_command(f"git checkout {MERGE_BRANCH}") + result = run_command(f"git pull --rebase upstream {UPSTREAM_BRANCH}") - if "Current branch is up to date" in result: - print("No changes detected from upstream.") - return False + if "Current branch is up to date" in result: + print("No changes detected from upstream.") + return - print("Pushing rebased changes to origin...") - run_command(f"git push origin {MERGE_BRANCH} --force") - return True + print("Pushing rebased changes to origin...") + run_command(f"git push origin {MERGE_BRANCH} --force") def detect_commits() -> list[str]: """Detect commits from upstream not yet in downstream.""" print("Detecting new commits from upstream...") - return run_command(f"git log {TARGET_BRANCH}..{MERGE_BRANCH} --pretty=format:'%h %s %b'").split("\n") + commit_log = run_command(f"git log {TARGET_BRANCH}..{MERGE_BRANCH} --pretty=format:'%h %s'").split("\n") + commit_log.reverse() + return commit_log def fetch_pull_body(pull_id) -> str | None: @@ -104,17 +110,31 @@ def fetch_pull_body(pull_id) -> str | None: def build_changelog(commit_log: list[str]) -> dict: """Generate the changelog from parsed commits.""" + print("Building changelog...") translator = Translator() changelog = {} + pull_cache = {} with ThreadPoolExecutor() as executor: futures = {} for commit in commit_log: - pull = re.search("#\d+", commit).group() - if not pull: + pull_match = re.search("#\\d+", commit) + if not pull_match: + print(f"Skipping {commit}") + continue + + pull_id = pull_match.group()[1:] + + if pull_id in pull_cache: + print( + f"WARNING: pull duplicate found.\n" + f"1: {pull_cache[pull_id]}\n" + f"2: {commit}" + ) + print(f"Skipping {commit}") continue - pull_id = pull[1:] + pull_cache[pull_id] = commit futures[executor.submit(fetch_pull_body, pull_id)] = pull_id for future in as_completed(futures): @@ -126,20 +146,28 @@ def build_changelog(commit_log: list[str]) -> dict: if not pull_body: continue - parsed = changelog_utils.parse_changelog(pull_body) - if parsed and parsed["changes"]: - for change in parsed["changes"]: - tag = change["tag"] - message = change["message"] - if TRANSLATE_CHANGES: - translated_message = translator.translate(message, src="en", dest="ru").text - pull_changes.append(f"{tag}: {translated_message}") - else: - pull_changes.append(f"{tag}: {message}") - pull_changes.append(f"") - - if pull_changes: - changelog[pull] = pull_changes + try: + parsed = changelog_utils.parse_changelog(pull_body) + if parsed and parsed["changes"]: + for change in parsed["changes"]: + tag = change["tag"] + message = change["message"] + if TRANSLATE_CHANGES: + translated_message = translator.translate(message, src="en", dest="ru").text + change = f"{tag}: {translated_message} " + else: + change = f"{tag}: {message} " + pull_changes.append(change) + + if pull_changes: + changelog[pull_id] = pull_changes + except Exception as e: + print( + f"An error occurred while processing {commit}\n" + f"URL: {pull_url}\n" + f"Body: {pull_body}" + ) + raise e return changelog @@ -148,25 +176,24 @@ def prepare_pull_body(changelog: dict) -> str: """Build new pull request body from the generated changelog.""" pull_body = ( f"This pull request merges upstream/{UPSTREAM_BRANCH}. " - f"Resolve possible conflicts manually and make sure all the changes applies correctly.\n" + f"Resolve possible conflicts manually and make sure all the changes are applied correctly.\n" ) if not changelog or not changelog.items(): return pull_body - pull_body += ( - f"\n## Changelog\n" - f":cl:\n" - ) - for change in changelog.values(): - pull_body += f"{change}\n" - pull_body += f"/:cl:\n" + pull_body += f"\n## Changelog\n" + pull_body += f":cl: {CHANGELOG_AUTHOR}\n" if CHANGELOG_AUTHOR else ":cl:\n" + for pull_changes in changelog.values(): + pull_body += f"{'\n'.join(pull_changes)}\n" + pull_body += "/:cl:\n" return pull_body def create_pr(pull_body: str): """Create a pull request with the processed changelog.""" + print("Creating pull request...") github = Github(GITHUB_TOKEN) repo = github.get_repo(TARGET_REPO) @@ -174,16 +201,21 @@ def create_pr(pull_body: str): repo.create_pull( title=f"Merge Upstream {datetime.today().strftime('%d.%m.%Y')}", body=pull_body, - head=TARGET_BRANCH, - base=MERGE_BRANCH + head=MERGE_BRANCH, + base=TARGET_BRANCH ) + print("Pull request created successfully.") if __name__ == "__main__": setup_repo() - if update_merge_branch(): - commit_log = detect_commits() + update_merge_branch() + commit_log = detect_commits() + + if commit_log: changelog = build_changelog(commit_log) pull_body = prepare_pull_body(changelog) create_pr(pull_body) + else: + print(f"No changes detected from {UPSTREAM_REPO}/{UPSTREAM_BRANCH}. Skipping pull request creation.") From 67d3ce49081a4815a853de168d13a4bde59599e7 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Tue, 10 Dec 2024 01:47:17 +0300 Subject: [PATCH 3/9] Implement label tracking --- tools/merge-upstream/merge_upstream.py | 112 ++++++++++++++++++------- 1 file changed, 80 insertions(+), 32 deletions(-) diff --git a/tools/merge-upstream/merge_upstream.py b/tools/merge-upstream/merge_upstream.py index 801248706c4d..264f1c7d08ae 100644 --- a/tools/merge-upstream/merge_upstream.py +++ b/tools/merge-upstream/merge_upstream.py @@ -5,10 +5,30 @@ from datetime import datetime from concurrent.futures import ThreadPoolExecutor, as_completed from github import Github +from github.PullRequest import PullRequest from googletrans import Translator import changelog_utils +CHANGELOG = "changelog" +MERGE_ORDER = "merge_order" +UPSTREAM_CONFIG_CHANGE_LABEL = "Configuration Change" +UPSTREAM_SQL_CHANGE_LABEL = "SQL Change" +UPSTREAM_WIKI_CHANGE_LABEL = "Requires Wiki Update" +DOWNSTREAM_WIKI_CHANGE_LABEL = ":page_with_curl: Требуется изменение WIKI" + +TRACKED_LABELS = { + UPSTREAM_CONFIG_CHANGE_LABEL, + UPSTREAM_SQL_CHANGE_LABEL, + UPSTREAM_WIKI_CHANGE_LABEL, +} + +LABEL_BLOCK_STYLE = { + UPSTREAM_CONFIG_CHANGE_LABEL: "IMPORTANT", + UPSTREAM_SQL_CHANGE_LABEL: "IMPORTANT", + UPSTREAM_WIKI_CHANGE_LABEL: "NOTE", +} + def check_env(): """Check if the required environment variables are set.""" @@ -90,16 +110,15 @@ def detect_commits() -> list[str]: return commit_log -def fetch_pull_body(pull_id) -> str | None: - """Fetch the pull request body from GitHub.""" +def fetch_pull(pull_id) -> PullRequest | None: + """Fetch the pull request from GitHub.""" github = Github(GITHUB_TOKEN) repo = github.get_repo(UPSTREAM_REPO) max_retries = 3 for attempt in range(max_retries): try: - pr = repo.get_pull(int(pull_id)) - return pr.body + return repo.get_pull(int(pull_id)) except Exception as e: print(f"Error fetching PR #{pull_id}: {e}") if attempt + 1 < max_retries: @@ -108,22 +127,26 @@ def fetch_pull_body(pull_id) -> str | None: return None -def build_changelog(commit_log: list[str]) -> dict: - """Generate the changelog from parsed commits.""" - print("Building changelog...") - translator = Translator() - changelog = {} +def build_details(commit_log: list[str]) -> dict: + """Generate data from parsed commits.""" + print("Building details...") + details = { + CHANGELOG: {}, + MERGE_ORDER: [match.group()[1:] for c in commit_log if (match := re.search("#\\d+", c))], + **{label: {} for label in TRACKED_LABELS} + } pull_cache = {} + translator = Translator() with ThreadPoolExecutor() as executor: futures = {} for commit in commit_log: - pull_match = re.search("#\\d+", commit) - if not pull_match: + match = re.search("#\\d+", commit) + if not match: print(f"Skipping {commit}") continue - pull_id = pull_match.group()[1:] + pull_id = match.group()[1:] if pull_id in pull_cache: print( @@ -135,75 +158,101 @@ def build_changelog(commit_log: list[str]) -> dict: continue pull_cache[pull_id] = commit - futures[executor.submit(fetch_pull_body, pull_id)] = pull_id + futures[executor.submit(fetch_pull, pull_id)] = pull_id for future in as_completed(futures): pull_id = futures[future] - pull_body = future.result() - pull_url = f"https://github.com/{UPSTREAM_REPO}/pull/{pull_id}" + pull: PullRequest | None = future.result() + labels = [label.name for label in pull.get_labels()] pull_changes = [] - if not pull_body: + if not pull: continue try: - parsed = changelog_utils.parse_changelog(pull_body) + for label in labels: + if label in TRACKED_LABELS: + details[label][pull_id] = pull + + parsed = changelog_utils.parse_changelog(pull.body) if parsed and parsed["changes"]: for change in parsed["changes"]: tag = change["tag"] message = change["message"] if TRANSLATE_CHANGES: translated_message = translator.translate(message, src="en", dest="ru").text - change = f"{tag}: {translated_message} " + change = f"{tag}: {translated_message} " else: - change = f"{tag}: {message} " + change = f"{tag}: {message} " pull_changes.append(change) if pull_changes: - changelog[pull_id] = pull_changes + details[CHANGELOG][pull_id] = pull_changes except Exception as e: print( f"An error occurred while processing {commit}\n" - f"URL: {pull_url}\n" - f"Body: {pull_body}" + f"URL: {pull.html_url}\n" + f"Body: {pull.body}" ) raise e - return changelog + return details -def prepare_pull_body(changelog: dict) -> str: +def prepare_pull_body(details: dict) -> str: """Build new pull request body from the generated changelog.""" pull_body = ( f"This pull request merges upstream/{UPSTREAM_BRANCH}. " f"Resolve possible conflicts manually and make sure all the changes are applied correctly.\n" ) - if not changelog or not changelog.items(): + if not details: + return pull_body + + for label in TRACKED_LABELS: + if not details[label]: + continue + + pull_body += ( + f"\n> [!{LABEL_BLOCK_STYLE[label]}]\n" + f"> {label}:\n" + ) + for _, pull in sorted(details[label].items()): + pull_body += f"> {pull.html_url}\n" + + if not details[CHANGELOG]: return pull_body pull_body += f"\n## Changelog\n" pull_body += f":cl: {CHANGELOG_AUTHOR}\n" if CHANGELOG_AUTHOR else ":cl:\n" - for pull_changes in changelog.values(): - pull_body += f"{'\n'.join(pull_changes)}\n" + for pull_id in details[MERGE_ORDER]: + if pull_id not in details[CHANGELOG]: + continue + pull_body += f"{'\n'.join(details[CHANGELOG][pull_id])}\n" pull_body += "/:cl:\n" return pull_body -def create_pr(pull_body: str): +def create_pr(details: dict): """Create a pull request with the processed changelog.""" + pull_body = prepare_pull_body(details) + print("Creating pull request...") github = Github(GITHUB_TOKEN) repo = github.get_repo(TARGET_REPO) # Create the pull request - repo.create_pull( + pull = repo.create_pull( title=f"Merge Upstream {datetime.today().strftime('%d.%m.%Y')}", body=pull_body, head=MERGE_BRANCH, base=TARGET_BRANCH ) + + if details[UPSTREAM_WIKI_CHANGE_LABEL]: + pull.add_to_labels(DOWNSTREAM_WIKI_CHANGE_LABEL) + print("Pull request created successfully.") @@ -214,8 +263,7 @@ def create_pr(pull_body: str): commit_log = detect_commits() if commit_log: - changelog = build_changelog(commit_log) - pull_body = prepare_pull_body(changelog) - create_pr(pull_body) + details = build_details(commit_log) + create_pr(details) else: print(f"No changes detected from {UPSTREAM_REPO}/{UPSTREAM_BRANCH}. Skipping pull request creation.") From 8774673ca2bfd8c78715b7526d0b38540e23b777 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Tue, 10 Dec 2024 03:34:01 +0300 Subject: [PATCH 4/9] Generate app token --- .github/workflows/merge_upstream.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/merge_upstream.yml b/.github/workflows/merge_upstream.yml index 339bc405deab..3f07d729d9c0 100644 --- a/.github/workflows/merge_upstream.yml +++ b/.github/workflows/merge_upstream.yml @@ -8,6 +8,14 @@ jobs: runs-on: ubuntu-latest steps: + - id: create_token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.PRIVATE_KEY }} + + - run: echo "GH_TOKEN=${{ steps.create_token.outputs.token }}" >> "$GITHUB_ENV" + - name: Set up Python uses: actions/setup-python@v4 with: @@ -25,7 +33,7 @@ jobs: - name: Run the script env: - GITHUB_TOKEN: ${{ secrets.BOT_TOKEN }} + GITHUB_TOKEN: ${{ env.GH_TOKEN }} TARGET_REPO: 'ss220club/Paradise-SS220' TARGET_BRANCH: 'master' UPSTREAM_REPO: 'ParadiseSS13/Paradise' From b256b6887cc4273f68a4b3e3f37f197b21e1547f Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Tue, 10 Dec 2024 03:47:55 +0300 Subject: [PATCH 5/9] Remove HTML comments from PR body --- tools/changelog/changelog_utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/changelog/changelog_utils.py b/tools/changelog/changelog_utils.py index 2f73343d872f..13bae286cc98 100644 --- a/tools/changelog/changelog_utils.py +++ b/tools/changelog/changelog_utils.py @@ -8,7 +8,7 @@ DISCORD_EMBED_DESCRIPTION_LIMIT = 4096 CL_BODY = re.compile(r"(:cl:|🆑)[ \t]*(?P.+?)?\s*\n(?P(.|\n)*?)\n/(:cl:|🆑)", re.MULTILINE) -CL_SPLIT = re.compile(r"\s*(?:)?((?P\w+)\s*:)?\s*(?P.*)") +CL_SPLIT = re.compile(r"\s*(?:(?P\w+)\s*:)?\s*(?P.*)") DISCORD_TAG_EMOJI = { "soundadd": ":notes:", @@ -55,8 +55,9 @@ def validate_changelog(changelog: dict): raise Exception(f"The changelog exceeds the length limit ({DISCORD_EMBED_DESCRIPTION_LIMIT}). Shorten it.") -def parse_changelog(message: str, tags_config: dict = {}) -> dict | None: - cl_parse_result = CL_BODY.search(message) +def parse_changelog(pr_body: str, tags_config: dict | None = None) -> dict | None: + clean_pr_body = re.sub(r"", "", pr_body, flags=re.DOTALL) + cl_parse_result = CL_BODY.search(clean_pr_body) if cl_parse_result is None: return None @@ -69,10 +70,14 @@ def parse_changelog(message: str, tags_config: dict = {}) -> dict | None: raise Exception(f"Invalid change: '{cl_line}'") tag = change_parse_result["tag"] message = change_parse_result["message"] + if tags_config and tag and tag not in tags_config['tags'].keys(): raise Exception(f"Invalid tag: '{cl_line}'. Valid tags: {', '.join(tags_config['tags'].keys())}") if not message: raise Exception(f"No message for change: '{cl_line}'") + + message = message.strip() + if tags_config and message in list(tags_config['defaults'].values()): # Check to see if the tags are associated with something that isn't the default text raise Exception(f"Don't use default message for change: '{cl_line}'") if tag: From 9efe884a781174b8b7310e14efa7aa9fd3876b24 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Tue, 10 Dec 2024 04:00:22 +0300 Subject: [PATCH 6/9] Make changelog check run anywhere --- .github/workflows/check_changelog.yml | 8 +++++--- tools/changelog/check_changelog.py | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check_changelog.yml b/.github/workflows/check_changelog.yml index ddff67582a83..c19d6fc7800a 100644 --- a/.github/workflows/check_changelog.yml +++ b/.github/workflows/check_changelog.yml @@ -12,12 +12,14 @@ on: jobs: CheckCL: runs-on: ubuntu-latest - if: github.repository == 'ss220club/Paradise-SS220' && github.base_ref == 'master' && github.event.pull_request.draft == false + if: github.base_ref == 'master' && github.event.pull_request.draft == false + steps: - name: Downloading scripts run: | - wget https://raw.githubusercontent.com/ss220club/Paradise-SS220/master/tools/changelog/check_changelog.py - wget https://raw.githubusercontent.com/ss220club/Paradise-SS220/master/tools/changelog/tags.yml + wget https://raw.githubusercontent.com/${{ github.repository }}/${{ github.base_ref }}/tools/changelog/changelog_utils.py + wget https://raw.githubusercontent.com/${{ github.repository }}/${{ github.base_ref }}/tools/changelog/check_changelog.py + wget https://raw.githubusercontent.com/${{ github.repository }}/${{ github.base_ref }}/tools/changelog/tags.yml - name: Installing Python uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 with: diff --git a/tools/changelog/check_changelog.py b/tools/changelog/check_changelog.py index 944bae6a57a3..3c0e28c20c62 100644 --- a/tools/changelog/check_changelog.py +++ b/tools/changelog/check_changelog.py @@ -2,10 +2,10 @@ DO NOT MANUALLY RUN THIS SCRIPT. --------------------------------- -Expected envrionmental variables: +Expected environmental variables: ----------------------------------- GITHUB_REPOSITORY: Github action variable representing the active repo (Action provided) -BOT_TOKEN: A repository account token, this will allow the action to push the changes (Action provided) +GITHUB_TOKEN: A repository account token, this will allow the action to push the changes (Action provided) GITHUB_EVENT_PATH: path to JSON file containing the event info (Action provided) """ import os @@ -18,7 +18,7 @@ # Blessed is the GoOnStAtIoN birb ZeWaKa for thinking of this first repo = os.getenv("GITHUB_REPOSITORY") -token = os.getenv("BOT_TOKEN") +token = os.getenv("GITHUB_TOKEN") event_path = os.getenv("GITHUB_EVENT_PATH") with open(event_path, 'r') as f: From cdf1f51e43663ca378537c37e86f46a746488e60 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Tue, 10 Dec 2024 15:54:17 +0300 Subject: [PATCH 7/9] Use reset instead of rebase --- tools/merge-upstream/merge_upstream.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tools/merge-upstream/merge_upstream.py b/tools/merge-upstream/merge_upstream.py index 264f1c7d08ae..02a0f0c46eb8 100644 --- a/tools/merge-upstream/merge_upstream.py +++ b/tools/merge-upstream/merge_upstream.py @@ -90,15 +90,11 @@ def update_merge_branch(): run_command(f"git push -u origin {MERGE_BRANCH}") return - print(f"Rebasing {MERGE_BRANCH} onto upstream/{UPSTREAM_BRANCH}...") + print(f"Resetting {MERGE_BRANCH} onto upstream/{UPSTREAM_BRANCH}...") run_command(f"git checkout {MERGE_BRANCH}") - result = run_command(f"git pull --rebase upstream {UPSTREAM_BRANCH}") + result = run_command(f"git reset --hard upstream/{UPSTREAM_BRANCH}") - if "Current branch is up to date" in result: - print("No changes detected from upstream.") - return - - print("Pushing rebased changes to origin...") + print("Pushing changes to origin...") run_command(f"git push origin {MERGE_BRANCH} --force") From 023df594457c6b61cf7932371a225b40e15c3ec5 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Tue, 10 Dec 2024 17:57:07 +0300 Subject: [PATCH 8/9] Enhance typing --- tools/merge-upstream/merge_upstream.py | 93 ++++++++++++++++---------- 1 file changed, 56 insertions(+), 37 deletions(-) diff --git a/tools/merge-upstream/merge_upstream.py b/tools/merge-upstream/merge_upstream.py index 02a0f0c46eb8..a5d4b108d682 100644 --- a/tools/merge-upstream/merge_upstream.py +++ b/tools/merge-upstream/merge_upstream.py @@ -1,7 +1,9 @@ +import enum import os import re import subprocess import time +import typing from datetime import datetime from concurrent.futures import ThreadPoolExecutor, as_completed from github import Github @@ -10,23 +12,29 @@ import changelog_utils -CHANGELOG = "changelog" -MERGE_ORDER = "merge_order" -UPSTREAM_CONFIG_CHANGE_LABEL = "Configuration Change" -UPSTREAM_SQL_CHANGE_LABEL = "SQL Change" -UPSTREAM_WIKI_CHANGE_LABEL = "Requires Wiki Update" -DOWNSTREAM_WIKI_CHANGE_LABEL = ":page_with_curl: Требуется изменение WIKI" - -TRACKED_LABELS = { - UPSTREAM_CONFIG_CHANGE_LABEL, - UPSTREAM_SQL_CHANGE_LABEL, - UPSTREAM_WIKI_CHANGE_LABEL, -} + +class UpstreamLabel(str, enum.Enum): + CONFIG_CHANGE = "Configuration Change" + SQL_CHANGE = "SQL Change" + WIKI_CHANGE = "Requires Wiki Update" + + +class DownstreamLabel(str, enum.Enum): + WIKI_CHANGE = ":page_with_curl: Требуется изменение WIKI" + + +class PullDetails(typing.TypedDict): + changelog: typing.Dict[str, list[str]] + merge_oder: list[str] + config_changes: typing.Dict[str, PullRequest] + sql_changes: typing.Dict[str, PullRequest] + wiki_changes: typing.Dict[str, PullRequest] + LABEL_BLOCK_STYLE = { - UPSTREAM_CONFIG_CHANGE_LABEL: "IMPORTANT", - UPSTREAM_SQL_CHANGE_LABEL: "IMPORTANT", - UPSTREAM_WIKI_CHANGE_LABEL: "NOTE", + UpstreamLabel.CONFIG_CHANGE: "IMPORTANT", + UpstreamLabel.SQL_CHANGE: "IMPORTANT", + UpstreamLabel.WIKI_CHANGE: "NOTE", } @@ -92,7 +100,7 @@ def update_merge_branch(): print(f"Resetting {MERGE_BRANCH} onto upstream/{UPSTREAM_BRANCH}...") run_command(f"git checkout {MERGE_BRANCH}") - result = run_command(f"git reset --hard upstream/{UPSTREAM_BRANCH}") + run_command(f"git reset --hard upstream/{UPSTREAM_BRANCH}") print("Pushing changes to origin...") run_command(f"git push origin {MERGE_BRANCH} --force") @@ -123,14 +131,16 @@ def fetch_pull(pull_id) -> PullRequest | None: return None -def build_details(commit_log: list[str]) -> dict: +def build_details(commit_log: list[str]) -> PullDetails: """Generate data from parsed commits.""" print("Building details...") - details = { - CHANGELOG: {}, - MERGE_ORDER: [match.group()[1:] for c in commit_log if (match := re.search("#\\d+", c))], - **{label: {} for label in TRACKED_LABELS} - } + details = PullDetails( + changelog={}, + merge_oder=[match.group()[1:] for c in commit_log if (match := re.search("#\\d+", c))], + config_changes={}, + sql_changes={}, + wiki_changes={} + ) pull_cache = {} translator = Translator() @@ -167,8 +177,12 @@ def build_details(commit_log: list[str]) -> dict: try: for label in labels: - if label in TRACKED_LABELS: - details[label][pull_id] = pull + if label == UpstreamLabel.CONFIG_CHANGE.value: + details["config_changes"][pull_id] = pull + elif label == UpstreamLabel.SQL_CHANGE.value: + details["sql_changes"][pull_id] = pull + elif label == UpstreamLabel.WIKI_CHANGE.value: + details["wiki_changes"][pull_id] = pull parsed = changelog_utils.parse_changelog(pull.body) if parsed and parsed["changes"]: @@ -183,7 +197,7 @@ def build_details(commit_log: list[str]) -> dict: pull_changes.append(change) if pull_changes: - details[CHANGELOG][pull_id] = pull_changes + details["changelog"][pull_id] = pull_changes except Exception as e: print( f"An error occurred while processing {commit}\n" @@ -195,7 +209,7 @@ def build_details(commit_log: list[str]) -> dict: return details -def prepare_pull_body(details: dict) -> str: +def prepare_pull_body(details: PullDetails) -> str: """Build new pull request body from the generated changelog.""" pull_body = ( f"This pull request merges upstream/{UPSTREAM_BRANCH}. " @@ -205,32 +219,37 @@ def prepare_pull_body(details: dict) -> str: if not details: return pull_body - for label in TRACKED_LABELS: - if not details[label]: + label_to_changes = { + UpstreamLabel.CONFIG_CHANGE: details["config_changes"], + UpstreamLabel.SQL_CHANGE: details["sql_changes"], + UpstreamLabel.WIKI_CHANGE: details["wiki_changes"] + } + for label, changes in label_to_changes.items(): + if not changes: continue pull_body += ( f"\n> [!{LABEL_BLOCK_STYLE[label]}]\n" - f"> {label}:\n" + f"> {label.value}:\n" ) - for _, pull in sorted(details[label].items()): + for _, pull in sorted(changes.items()): pull_body += f"> {pull.html_url}\n" - if not details[CHANGELOG]: + if not details["changelog"]: return pull_body pull_body += f"\n## Changelog\n" pull_body += f":cl: {CHANGELOG_AUTHOR}\n" if CHANGELOG_AUTHOR else ":cl:\n" - for pull_id in details[MERGE_ORDER]: - if pull_id not in details[CHANGELOG]: + for pull_id in details["merge_oder"]: + if pull_id not in details["changelog"]: continue - pull_body += f"{'\n'.join(details[CHANGELOG][pull_id])}\n" + pull_body += f"{'\n'.join(details["changelog"][pull_id])}\n" pull_body += "/:cl:\n" return pull_body -def create_pr(details: dict): +def create_pr(details: PullDetails): """Create a pull request with the processed changelog.""" pull_body = prepare_pull_body(details) @@ -246,8 +265,8 @@ def create_pr(details: dict): base=TARGET_BRANCH ) - if details[UPSTREAM_WIKI_CHANGE_LABEL]: - pull.add_to_labels(DOWNSTREAM_WIKI_CHANGE_LABEL) + if details["wiki_changes"]: + pull.add_to_labels(DownstreamLabel.WIKI_CHANGE) print("Pull request created successfully.") From e5f9faef12b840374e9347582f3bd3ad5237932f Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Tue, 10 Dec 2024 19:10:04 +0300 Subject: [PATCH 9/9] Fix typo --- tools/merge-upstream/merge_upstream.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/merge-upstream/merge_upstream.py b/tools/merge-upstream/merge_upstream.py index a5d4b108d682..3ddf79ebe6d7 100644 --- a/tools/merge-upstream/merge_upstream.py +++ b/tools/merge-upstream/merge_upstream.py @@ -25,7 +25,7 @@ class DownstreamLabel(str, enum.Enum): class PullDetails(typing.TypedDict): changelog: typing.Dict[str, list[str]] - merge_oder: list[str] + merge_order: list[str] config_changes: typing.Dict[str, PullRequest] sql_changes: typing.Dict[str, PullRequest] wiki_changes: typing.Dict[str, PullRequest] @@ -136,7 +136,7 @@ def build_details(commit_log: list[str]) -> PullDetails: print("Building details...") details = PullDetails( changelog={}, - merge_oder=[match.group()[1:] for c in commit_log if (match := re.search("#\\d+", c))], + merge_order=[match.group()[1:] for c in commit_log if (match := re.search("#\\d+", c))], config_changes={}, sql_changes={}, wiki_changes={} @@ -240,7 +240,7 @@ def prepare_pull_body(details: PullDetails) -> str: pull_body += f"\n## Changelog\n" pull_body += f":cl: {CHANGELOG_AUTHOR}\n" if CHANGELOG_AUTHOR else ":cl:\n" - for pull_id in details["merge_oder"]: + for pull_id in details["merge_order"]: if pull_id not in details["changelog"]: continue pull_body += f"{'\n'.join(details["changelog"][pull_id])}\n"