From 1ebd5cc34a75ffaf643ba162753fddf2da5c097e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Fri, 1 Nov 2024 17:24:43 +0100 Subject: [PATCH] Changelog: Skip some duplicated jobs --- github_app_geo_project/module/__init__.py | 8 + .../module/standard/changelog.py | 166 ++++++++++-------- github_app_geo_project/views/webhook.py | 44 ++++- 3 files changed, 142 insertions(+), 76 deletions(-) diff --git a/github_app_geo_project/module/__init__.py b/github_app_geo_project/module/__init__.py index 2b8267abb7c..47f6e0ecf4e 100644 --- a/github_app_geo_project/module/__init__.py +++ b/github_app_geo_project/module/__init__.py @@ -321,6 +321,14 @@ def documentation_url(self) -> str: """Get the URL to the documentation page of the module.""" return "" + def jobs_unique_on(self) -> list[str] | None: + """ + Return the list of fields that should be unique for the jobs. + + If not unique, the other jobs will be skipped. + """ + return None + @abstractmethod def get_actions(self, context: GetActionContext) -> list[Action[_EVENT_DATA]]: """ diff --git a/github_app_geo_project/module/standard/changelog.py b/github_app_geo_project/module/standard/changelog.py index f957b4ad834..962cbcff0c8 100644 --- a/github_app_geo_project/module/standard/changelog.py +++ b/github_app_geo_project/module/standard/changelog.py @@ -351,11 +351,13 @@ def generate_changelog( configuration: changelog_configuration.Changelog, repository: str, tag_str: str, - milestone: github.Milestone.Milestone, ) -> str: """Generate the changelog for a tag.""" repo = github_application.get_repo(repository) + milestones = [m for m in repo.get_milestones() if m.title == tag_str] + milestone = milestones[0] if milestones else repo.create_milestone(tag_str) + discussion_url = _get_discussion_url(repo, tag_str) tags: dict[Tag, Tag] = {} @@ -484,6 +486,10 @@ def documentation_url(self) -> str: """Get the URL to the documentation page of the module.""" return "https://github.com/camptocamp/github-app-geo-project/wiki/Module-%E2%80%90-Changelog" + def jobs_unique_on(self) -> list[str] | None: + """Indicate fields used to ship other jobs.""" + return ["repository", "owner", "module_data"] + def get_actions(self, context: module.GetActionContext) -> list[module.Action[dict[str, Any]]]: """ Get the action related to the module and the event. @@ -493,26 +499,70 @@ def get_actions(self, context: module.GetActionContext) -> list[module.Action[di """ event_data = context.event_data if "release" in event_data and event_data.get("action") == "created": - return [module.Action(priority=module.PRIORITY_STATUS, data={"type": "release"})] + return [ + module.Action( + priority=module.PRIORITY_STATUS, + data={"version": event_data["release"]["tag_name"]}, + ) + ] if event_data.get("ref_type") == "tag": - return [module.Action(priority=module.PRIORITY_STATUS, data={"type": "tag"})] + return [ + module.Action( + priority=module.PRIORITY_STATUS, data={"type": "tag", "version": event_data["ref"]} + ) + ] if ( - ( - event_data.get("action") == "edited" - or event_data.get("action") == "labeled" - or event_data.get("action") == "unlabeled" - or event_data.get("action") == "milestoned" - ) + event_data.get("action") in ("edited", "labelled", "unlabelled", "milestoned", "demilestoned") and event_data.get("pull_request", {}).get("state") == "closed" - and event_data.get("pull_request", {}).get("milestone") and event_data.get("sender", {}).get("login") != context.github_application.integration.get_app().slug + "[bot]" ): - return [module.Action(priority=module.PRIORITY_CRON, data={"type": "pull_request"})] - if event_data.get("action") == "edited" and "milestone" in event_data: - return [module.Action(priority=module.PRIORITY_CRON, data={"type": "milestone"})] + versions = set() + milestone_version = event_data.get("milestone", {}).get("title") + if milestone_version is not None: + versions.add(milestone_version) + pull_request_version = event_data.get("pull_request", {}).get("milestone", {}).get("title") + if pull_request_version is not None: + versions.add(pull_request_version) + return [ + module.Action( + priority=module.PRIORITY_CRON, + data={"version": version}, + ) + for version in versions + ] + + if ( + event_data.get("action") == "edited" + and "milestone" in event_data + and event_data.get("sender", {}).get("login") + != context.github_application.integration.get_app().slug + "[bot]" + ): + versions = {event_data["milestone"]["title"]} + if "changes" in event_data and "title" in event_data["changes"]: + versions.add(event_data["changes"]["title"]["from"]) + + return [ + module.Action( + priority=module.PRIORITY_CRON, + data={"version": version}, + ) + for version in versions + ] if event_data.get("action") in ("created", "closed") and "discussion" in event_data: - return [module.Action(priority=module.PRIORITY_CRON, data={"type": "discussion"})] + return [ + module.Action( + priority=module.PRIORITY_STATUS, + data={"type": "discussion"}, + ) + ] + if event_data.get("action") == "edited" and "title" in event_data.get("changes", {}): + return [ + module.Action( + priority=module.PRIORITY_STATUS, + data={"type": "discussion"}, + ) + ] return [] @@ -538,16 +588,13 @@ async def process( description=config.get("description", ""), ) - assert isinstance(repository, str) - tag_str = "" - milestone = None - release = None + tag_str = cast(str, context.module_event_data.get("version")) if context.module_event_data.get("type") == "tag": if not context.module_config.get( "create-release", changelog_configuration.CREATE_RELEASE_DEFAULT ): return module.ProcessOutput() - tag_str = cast(str, context.event_data["ref"]) + prerelease = False try: latest_release = repo.get_latest_release() @@ -558,26 +605,20 @@ async def process( except github.UnknownObjectException as exception: if exception.status != 404: raise - release = repo.create_git_release(tag_str, tag_str, "", prerelease=prerelease) - elif context.module_event_data.get("type") == "release": - if context.module_config.get("create-release", changelog_configuration.CREATE_RELEASE_DEFAULT): - return module.ProcessOutput() - tag_str = context.event_data.get("release", {}).get("tag_name") - release = repo.get_release(tag_str) - elif context.module_event_data.get("type") == "milestone": - tag_str = context.event_data.get("milestone", {}).get("title") - tags = [tag for tag in repo.get_tags() if tag.name == tag_str] - if not tags: - _LOGGER.info( - "No tag found via for milestone %s on repository %s", - context.event_data.get("milestone", {}).get("title"), - repository, - ) - return module.ProcessOutput() - tag = tags[0] - release = repo.get_release(tag_str) + repo.create_git_release(tag_str, tag_str, "", prerelease=prerelease) + return module.ProcessOutput( + actions=[ + module.Action( + priority=module.PRIORITY_CRON, + data={"version": tag_str}, + ) + ] + ) elif context.module_event_data.get("type") == "discussion": - title = context.event_data.get("discussion", {}).get("title", "").split() + title = set() + title.update(context.event_data.get("discussion", {}).get("title", "").split()) + if "title" in context.event_data.get("changes", {}): + title.update(context.event_data["changes"]["title"]["from"].split()) tags = [tag for tag in repo.get_tags() if tag.name in title] if not tags: _LOGGER.info( @@ -586,48 +627,27 @@ async def process( repository, ) return module.ProcessOutput() - tag = tags[0] - tag_str = tag.name - release = repo.get_release(tag_str) - elif context.module_event_data.get("type") == "pull_request": - # Get the milestone - tag_str = context.event_data.get("pull_request", {}).get("milestone", {}).get("title") - try: - release = repo.get_release(tag_str) - except github.UnknownObjectException as exception: - if exception.status == 404: - # Create a release - prerelease = False - latest_release = repo.get_latest_release() - if latest_release is not None: - prerelease = packaging.version.Version(tag_str) < packaging.version.Version( - latest_release.tag_name - ) - release = repo.create_git_release(tag_str, tag_str, "", prerelease=prerelease) - else: - raise - tag = [tag for tag in repo.get_tags() if tag.name == tag_str][0] - if tag is None: - _LOGGER.info( - "No tag found via the milestone for pull request %s on repository %s", - context.event_data.get("pull_request", {}).get("number"), - repository, - ) - return module.ProcessOutput() + return module.ProcessOutput( + actions=[ + module.Action( + priority=module.PRIORITY_CRON, + data={"version": tags[0].name}, + ) + ] + ) - if milestone is None: - milestones = [m for m in repo.get_milestones() if m.title == tag_str] - if milestones: - milestone = milestones[0] - else: - milestone = repo.create_milestone(tag_str) + tags = [tag for tag in repo.get_tags() if tag.name == tag_str] + if not tags: + _LOGGER.info("No tag found '%s' on repository '%s'.", tag_str, repository) + return module.ProcessOutput() + release = repo.get_release(tag_str) assert release is not None release.update_release( tag_str, tag_name=tag_str, message=generate_changelog( - context.github_project.github, context.module_config, repository, tag_str, milestone + context.github_project.github, context.module_config, repository, tag_str ), ) return module.ProcessOutput() diff --git a/github_app_geo_project/views/webhook.py b/github_app_geo_project/views/webhook.py index 5e7e5f8dafa..427905a7e4e 100644 --- a/github_app_geo_project/views/webhook.py +++ b/github_app_geo_project/views/webhook.py @@ -11,6 +11,7 @@ import pyramid.request import sqlalchemy.engine import sqlalchemy.orm +from numpy import where from pyramid.view import view_config from sqlalchemy.orm import Session @@ -229,15 +230,52 @@ def process_event(context: ProcessContext) -> None: github_application=context.github_application, ) ): + priority = action.priority if action.priority >= 0 else module.PRIORITY_STANDARD + event_name = action.title or context.event_name + module_data = current_module.event_data_to_json(action.data) + + jobs_unique_on = current_module.jobs_unique_on() + if jobs_unique_on: + + update = ( + sqlalchemy.update(models.Queue) + .where(models.Queue.status == models.JobStatus.NEW) + .where(models.Queue.application == context.application) + .where(models.Queue.module == name) + ) + for key in jobs_unique_on: + if key == "priority": + update = update.where(models.Queue.priority == priority) + elif key == "owner": + update = update.where(models.Queue.owner == context.owner) + elif key == "repository": + update = update.where(models.Queue.repository == context.repository) + elif key == "event_name": + update = update.where(models.Queue.event_name == event_name) + elif key == "event_data": + update = update.where(models.Queue.event_data == context.event_data) + elif key == "module_data": + update = update.where(models.Queue.module_data == module_data) + else: + _LOGGER.error("Unknown jobs_unique_on key: %s", key) + + update = update.values( + { + "status": models.JobStatus.SKIPPED, + } + ) + + context.session.execute(update) + job = models.Queue() - job.priority = action.priority if action.priority >= 0 else module.PRIORITY_STANDARD + job.priority = priority job.application = context.application job.owner = context.owner job.repository = context.repository - job.event_name = action.title or context.event_name + job.event_name = event_name job.event_data = context.event_data job.module = name - job.module_data = current_module.event_data_to_json(action.data) + job.module_data = module_data context.session.add(job) context.session.flush()