Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Changelog: Skip some duplicated jobs #582

Merged
merged 1 commit into from
Nov 5, 2024
Merged

Changelog: Skip some duplicated jobs #582

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions github_app_geo_project/module/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,14 @@ def documentation_url(self) -> str:
"""Get the URL to the documentation page of the module."""
return ""

def jobs_unique_on(self) -> list[str] | None:
"""
Return the list of fields that should be unique for the jobs.

If not unique, the other jobs will be skipped.
"""
return None

@abstractmethod
def get_actions(self, context: GetActionContext) -> list[Action[_EVENT_DATA]]:
"""
Expand Down
166 changes: 93 additions & 73 deletions github_app_geo_project/module/standard/changelog.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -351,11 +351,13 @@ def generate_changelog(
configuration: changelog_configuration.Changelog,
repository: str,
tag_str: str,
milestone: github.Milestone.Milestone,
) -> str:
"""Generate the changelog for a tag."""
repo = github_application.get_repo(repository)

milestones = [m for m in repo.get_milestones() if m.title == tag_str]
milestone = milestones[0] if milestones else repo.create_milestone(tag_str)

discussion_url = _get_discussion_url(repo, tag_str)

tags: dict[Tag, Tag] = {}
Expand Down Expand Up @@ -484,6 +486,10 @@ def documentation_url(self) -> str:
"""Get the URL to the documentation page of the module."""
return "https://github.com/camptocamp/github-app-geo-project/wiki/Module-%E2%80%90-Changelog"

def jobs_unique_on(self) -> list[str] | None:
"""Indicate fields used to ship other jobs."""
return ["repository", "owner", "module_data"]

def get_actions(self, context: module.GetActionContext) -> list[module.Action[dict[str, Any]]]:
"""
Get the action related to the module and the event.
Expand All @@ -493,26 +499,70 @@ def get_actions(self, context: module.GetActionContext) -> list[module.Action[di
"""
event_data = context.event_data
if "release" in event_data and event_data.get("action") == "created":
return [module.Action(priority=module.PRIORITY_STATUS, data={"type": "release"})]
return [
module.Action(
priority=module.PRIORITY_STATUS,
data={"version": event_data["release"]["tag_name"]},
)
]
if event_data.get("ref_type") == "tag":
return [module.Action(priority=module.PRIORITY_STATUS, data={"type": "tag"})]
return [
module.Action(
priority=module.PRIORITY_STATUS, data={"type": "tag", "version": event_data["ref"]}
)
]
if (
(
event_data.get("action") == "edited"
or event_data.get("action") == "labeled"
or event_data.get("action") == "unlabeled"
or event_data.get("action") == "milestoned"
)
event_data.get("action") in ("edited", "labelled", "unlabelled", "milestoned", "demilestoned")
and event_data.get("pull_request", {}).get("state") == "closed"
and event_data.get("pull_request", {}).get("milestone")
and event_data.get("sender", {}).get("login")
!= context.github_application.integration.get_app().slug + "[bot]"
):
return [module.Action(priority=module.PRIORITY_CRON, data={"type": "pull_request"})]
if event_data.get("action") == "edited" and "milestone" in event_data:
return [module.Action(priority=module.PRIORITY_CRON, data={"type": "milestone"})]
versions = set()
milestone_version = event_data.get("milestone", {}).get("title")
if milestone_version is not None:
versions.add(milestone_version)
pull_request_version = event_data.get("pull_request", {}).get("milestone", {}).get("title")
if pull_request_version is not None:
versions.add(pull_request_version)
return [
module.Action(
priority=module.PRIORITY_CRON,
data={"version": version},
)
for version in versions
]

if (
event_data.get("action") == "edited"
and "milestone" in event_data
and event_data.get("sender", {}).get("login")
!= context.github_application.integration.get_app().slug + "[bot]"
):
versions = {event_data["milestone"]["title"]}
if "changes" in event_data and "title" in event_data["changes"]:
versions.add(event_data["changes"]["title"]["from"])

return [
module.Action(
priority=module.PRIORITY_CRON,
data={"version": version},
)
for version in versions
]
if event_data.get("action") in ("created", "closed") and "discussion" in event_data:
return [module.Action(priority=module.PRIORITY_CRON, data={"type": "discussion"})]
return [
module.Action(
priority=module.PRIORITY_STATUS,
data={"type": "discussion"},
)
]
if event_data.get("action") == "edited" and "title" in event_data.get("changes", {}):
return [
module.Action(
priority=module.PRIORITY_STATUS,
data={"type": "discussion"},
)
]

return []

Expand All @@ -538,16 +588,13 @@ async def process(
description=config.get("description", ""),
)

assert isinstance(repository, str)
tag_str = ""
milestone = None
release = None
tag_str = cast(str, context.module_event_data.get("version"))
if context.module_event_data.get("type") == "tag":
if not context.module_config.get(
"create-release", changelog_configuration.CREATE_RELEASE_DEFAULT
):
return module.ProcessOutput()
tag_str = cast(str, context.event_data["ref"])

prerelease = False
try:
latest_release = repo.get_latest_release()
Expand All @@ -558,26 +605,20 @@ async def process(
except github.UnknownObjectException as exception:
if exception.status != 404:
raise
release = repo.create_git_release(tag_str, tag_str, "", prerelease=prerelease)
elif context.module_event_data.get("type") == "release":
if context.module_config.get("create-release", changelog_configuration.CREATE_RELEASE_DEFAULT):
return module.ProcessOutput()
tag_str = context.event_data.get("release", {}).get("tag_name")
release = repo.get_release(tag_str)
elif context.module_event_data.get("type") == "milestone":
tag_str = context.event_data.get("milestone", {}).get("title")
tags = [tag for tag in repo.get_tags() if tag.name == tag_str]
if not tags:
_LOGGER.info(
"No tag found via for milestone %s on repository %s",
context.event_data.get("milestone", {}).get("title"),
repository,
)
return module.ProcessOutput()
tag = tags[0]
release = repo.get_release(tag_str)
repo.create_git_release(tag_str, tag_str, "", prerelease=prerelease)
return module.ProcessOutput(
actions=[
module.Action(
priority=module.PRIORITY_CRON,
data={"version": tag_str},
)
]
)
elif context.module_event_data.get("type") == "discussion":
title = context.event_data.get("discussion", {}).get("title", "").split()
title = set()
title.update(context.event_data.get("discussion", {}).get("title", "").split())
if "title" in context.event_data.get("changes", {}):
title.update(context.event_data["changes"]["title"]["from"].split())
tags = [tag for tag in repo.get_tags() if tag.name in title]
if not tags:
_LOGGER.info(
Expand All @@ -586,48 +627,27 @@ async def process(
repository,
)
return module.ProcessOutput()
tag = tags[0]
tag_str = tag.name
release = repo.get_release(tag_str)
elif context.module_event_data.get("type") == "pull_request":
# Get the milestone
tag_str = context.event_data.get("pull_request", {}).get("milestone", {}).get("title")
try:
release = repo.get_release(tag_str)
except github.UnknownObjectException as exception:
if exception.status == 404:
# Create a release
prerelease = False
latest_release = repo.get_latest_release()
if latest_release is not None:
prerelease = packaging.version.Version(tag_str) < packaging.version.Version(
latest_release.tag_name
)
release = repo.create_git_release(tag_str, tag_str, "", prerelease=prerelease)
else:
raise
tag = [tag for tag in repo.get_tags() if tag.name == tag_str][0]
if tag is None:
_LOGGER.info(
"No tag found via the milestone for pull request %s on repository %s",
context.event_data.get("pull_request", {}).get("number"),
repository,
)
return module.ProcessOutput()
return module.ProcessOutput(
actions=[
module.Action(
priority=module.PRIORITY_CRON,
data={"version": tags[0].name},
)
]
)

if milestone is None:
milestones = [m for m in repo.get_milestones() if m.title == tag_str]
if milestones:
milestone = milestones[0]
else:
milestone = repo.create_milestone(tag_str)
tags = [tag for tag in repo.get_tags() if tag.name == tag_str]
if not tags:
_LOGGER.info("No tag found '%s' on repository '%s'.", tag_str, repository)
return module.ProcessOutput()

release = repo.get_release(tag_str)
assert release is not None
release.update_release(
tag_str,
tag_name=tag_str,
message=generate_changelog(
context.github_project.github, context.module_config, repository, tag_str, milestone
context.github_project.github, context.module_config, repository, tag_str
),
)
return module.ProcessOutput()
Expand Down
44 changes: 41 additions & 3 deletions github_app_geo_project/views/webhook.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import pyramid.request
import sqlalchemy.engine
import sqlalchemy.orm
from numpy import where
from pyramid.view import view_config
from sqlalchemy.orm import Session

Expand Down Expand Up @@ -229,15 +230,52 @@ def process_event(context: ProcessContext) -> None:
github_application=context.github_application,
)
):
priority = action.priority if action.priority >= 0 else module.PRIORITY_STANDARD
event_name = action.title or context.event_name
module_data = current_module.event_data_to_json(action.data)

jobs_unique_on = current_module.jobs_unique_on()
if jobs_unique_on:

update = (
sqlalchemy.update(models.Queue)
.where(models.Queue.status == models.JobStatus.NEW)
.where(models.Queue.application == context.application)
.where(models.Queue.module == name)
)
for key in jobs_unique_on:
if key == "priority":
update = update.where(models.Queue.priority == priority)
elif key == "owner":
update = update.where(models.Queue.owner == context.owner)
elif key == "repository":
update = update.where(models.Queue.repository == context.repository)
elif key == "event_name":
update = update.where(models.Queue.event_name == event_name)
elif key == "event_data":
update = update.where(models.Queue.event_data == context.event_data)
elif key == "module_data":
update = update.where(models.Queue.module_data == module_data)
else:
_LOGGER.error("Unknown jobs_unique_on key: %s", key)

update = update.values(
{
"status": models.JobStatus.SKIPPED,
}
)

context.session.execute(update)

job = models.Queue()
job.priority = action.priority if action.priority >= 0 else module.PRIORITY_STANDARD
job.priority = priority
job.application = context.application
job.owner = context.owner
job.repository = context.repository
job.event_name = action.title or context.event_name
job.event_name = event_name
job.event_data = context.event_data
job.module = name
job.module_data = current_module.event_data_to_json(action.data)
job.module_data = module_data
context.session.add(job)
context.session.flush()

Expand Down