From 4cceeb7e16038dd4396bc9b20d283bd19a2f023a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 9 Aug 2023 14:14:44 +0200 Subject: [PATCH 1/3] eessi_bot_event_handler.py: added comments for categories of imports + ordered imports alphabetically --- eessi_bot_event_handler.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/eessi_bot_event_handler.py b/eessi_bot_event_handler.py index c4a8c267..e5529ec8 100644 --- a/eessi_bot_event_handler.py +++ b/eessi_bot_event_handler.py @@ -13,24 +13,27 @@ # # license: GPLv2 # -import waitress + +# Standard library imports import sys -import tasks.build as build -import tasks.deploy as deploy +# Third party imports (anything installed into the local Python environment) +from pyghee.lib import create_app, get_event_info, PyGHee, read_event_from_json +from pyghee.utils import log +import waitress + +# Local application imports (anything from EESSI/eessi-bot-software-layer) from connections import github +import tasks.build as build from tasks.build import check_build_permission, get_architecture_targets, get_repo_cfg, submit_build_jobs +import tasks.deploy as deploy from tasks.deploy import deploy_built_artefacts from tools import config from tools.args import event_handler_parse from tools.commands import EESSIBotCommand, EESSIBotCommandError, get_bot_command -# from tools.filter import EESSIBotActionFilter from tools.permissions import check_command_permission from tools.pr_comments import create_comment -from pyghee.lib import PyGHee, create_app, get_event_info, read_event_from_json -from pyghee.utils import log - APP_NAME = "app_name" BOT_CONTROL = "bot_control" From 6b4faa30adf8ff8d0ad1ee6f3742c279c10e6151 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 9 Aug 2023 15:13:52 +0200 Subject: [PATCH 2/3] eessi_bot_event_handler.py: improved and completed docstrings --- eessi_bot_event_handler.py | 140 ++++++++++++++++++++++++++++++++----- 1 file changed, 123 insertions(+), 17 deletions(-) diff --git a/eessi_bot_event_handler.py b/eessi_bot_event_handler.py index e5529ec8..f993777c 100644 --- a/eessi_bot_event_handler.py +++ b/eessi_bot_event_handler.py @@ -43,10 +43,17 @@ class EESSIBotSoftwareLayer(PyGHee): + """ + Class for representing the event handler of the build-and-deploy bot. It + receives events from GitHub via PyGHee and processes them. It is + multi-threaded (via waitress) to ensure that it can respond to concurrent + events. It also avoids keeping any event related information in memory. + """ def __init__(self, *args, **kwargs): """ - EESSIBotSoftwareLayer constructor. + EESSIBotSoftwareLayer constructor. Calls constructor of PyGHee and + initializes some configuration settings. """ super(EESSIBotSoftwareLayer, self).__init__(*args, **kwargs) @@ -56,11 +63,15 @@ def __init__(self, *args, **kwargs): def log(self, msg, *args): """ - Logs a message incl the caller's function name by passing msg and *args to PyGHee's log method. + Logs a message incl the caller's function name by passing msg and + *args to PyGHee's log method. Args: - msg (string): message (format) to log to event handler log + msg (string): message to log to event handler log *args (any): any values to be substituted into msg + + Returns: + None (implicitly) """ funcname = sys._getframe().f_back.f_code.co_name if args: @@ -70,7 +81,18 @@ def log(self, msg, *args): def handle_issue_comment_event(self, event_info, log_file=None): """ - Handle adding/removing of comment in issue or PR. + Handle events of type issue_comment. Main action is to parse new issue + comments for any bot command and execute it if one is found. + + Args: + event_info (dict): event received by event_handler + log_file (string): path to log messages to + + Returns: + None (implicitly) + + Raises: + Exception: raises any exception that is not of type EESSIBotCommandError """ request_body = event_info['raw_request_body'] issue_url = request_body['issue']['url'] @@ -222,7 +244,14 @@ def handle_issue_comment_event(self, event_info, log_file=None): def handle_installation_event(self, event_info, log_file=None): """ - Handle installation of app. + Handle events of type installation. Main action is to log the event. + + Args: + event_info (dict): event received by event_handler + log_file (string): path to log messages to + + Returns: + None (implicitly) """ request_body = event_info['raw_request_body'] user = request_body['sender']['login'] @@ -233,7 +262,15 @@ def handle_installation_event(self, event_info, log_file=None): def handle_pull_request_labeled_event(self, event_info, pr): """ - Handle adding of a label to a pull request. + Handle events of type pull_request with the action labeled. Main action + is to process the label 'bot:deploy'. + + Args: + event_info (dict): event received by event_handler + pr (github.PullRequest.PullRequest): instance representing the pull request + + Returns: + None (implicitly) """ # determine label @@ -267,7 +304,17 @@ def handle_pull_request_labeled_event(self, event_info, pr): def handle_pull_request_opened_event(self, event_info, pr): """ - Handle opening of a pull request. + Handle events of type pull_request with the action opened. Main action + is to report for which architectures and repositories a bot instance is + configured to build for. + + Args: + event_info (dict): event received by event_handler + pr (github.PullRequest.PullRequest): instance representing the pull request + + Returns: + github.IssueComment.IssueComment instance or None (note, github refers to + PyGithub, not the github from the internal connections module) """ self.log("PR opened: waiting for label bot:build") app_name = self.cfg[GITHUB][APP_NAME] @@ -298,7 +345,15 @@ def handle_pull_request_opened_event(self, event_info, pr): def handle_pull_request_event(self, event_info, log_file=None): """ - Handle 'pull_request' event + Handle events of type pull_request for all kinds of actions by + determining a handler for it. + + Args: + event_info (dict): event received by event_handler + log_file (string): path to log messages to + + Returns: + None (implicitly) """ action = event_info['action'] gh = github.get_instance() @@ -317,11 +372,21 @@ def handle_pull_request_event(self, event_info, log_file=None): def handle_bot_command(self, event_info, bot_command, log_file=None): """ - Handle bot command + Handle a bot command. Main purpose is to determine a handler for the + specific bot_command given. Args: - event_info (dict): object containing all information of the event + event_info (dict): event received by event_handler bot_command (EESSIBotCommand): command to be handled + log_file (string): path to log messages to + + Returns: + (string): update to be reported back to GitHub as the (immediate) + result of the bot command + + Raises: + EESSIBotCommandError: if no handler for the specific command is + defined """ cmd = bot_command.command handler_name = f"handle_bot_command_{cmd}" @@ -334,7 +399,17 @@ def handle_bot_command(self, event_info, bot_command, log_file=None): raise EESSIBotCommandError(f"unknown command `{cmd}`; use `bot: help` for usage information") def handle_bot_command_help(self, event_info, bot_command): - """handles command 'bot: help' with a simple usage info""" + """ + Handles bot command 'help' providing basic information about bot + commands. + + Args: + event_info (dict): event received by event_handler + bot_command (EESSIBotCommand): command to be handled + + Returns: + (string): basic information about sending commands to the bot + """ help_msg = "\n **How to send commands to bot instances**" help_msg += "\n - Commands must be sent with a **new** comment (edits of existing comments are ignored)." help_msg += "\n - A comment may contain multiple commands, one per line." @@ -345,7 +420,17 @@ def handle_bot_command_help(self, event_info, bot_command): return help_msg def handle_bot_command_build(self, event_info, bot_command): - """handles command 'bot: build [ARGS*]' by parsing arguments and submitting jobs""" + """ + Handles bot command 'build [ARGS*]' by parsing arguments and submitting jobs + + Args: + event_info (dict): event received by event_handler + bot_command (EESSIBotCommand): command to be handled + + Returns: + (string): immediate result of command (any jobs or no jobs being + submitted) and a link to the issue comment for submitted jobs + """ gh = github.get_instance() self.log("repository: '%s'", event_info['raw_request_body']['repository']['full_name']) repo_name = event_info['raw_request_body']['repository']['full_name'] @@ -369,7 +454,18 @@ def handle_bot_command_build(self, event_info, bot_command): return build_msg def handle_bot_command_show_config(self, event_info, bot_command): - """handles command 'bot: show_config' by printing a list of configured build targets""" + """ + Handles bot command 'show_config' by running the handler for events of + type pull_request with the action opened. + + Args: + event_info (dict): event received by event_handler + bot_command (EESSIBotCommand): command to be handled + + Returns: + (string): list item with a link to the issue comment that was created + by the handler for events of type pull_request with the action opened + """ self.log("processing bot command 'show_config'") gh = github.get_instance() repo_name = event_info['raw_request_body']['repository']['full_name'] @@ -379,11 +475,17 @@ def handle_bot_command_show_config(self, event_info, bot_command): return f"\n - added comment {issue_comment.html_url} to show configuration" def start(self, app, port=3000): - """starts the app and log information in the log file + """ + Logs startup information to shell and log file and starts the app using + waitress. Args: - app (object): instance of class EESSIBotSoftwareLayer - port (int, optional): Defaults to 3000. + app (EESSIBotSoftwareLayer): instance of class EESSIBotSoftwareLayer + port (int, optional): defaults to 3000 + + Returns: + None (implictly), Note it only returns once the call to waitress has + terminated. """ start_msg = "EESSI bot for software layer started!" print(start_msg) @@ -401,7 +503,11 @@ def start(self, app, port=3000): def main(): - """Main function.""" + """ + Main function which parses command line arguments, verifies if required + configuration settings are defined, creates an instance of EESSIBotSoftwareLayer + and starts it. + """ opts = event_handler_parse() required_config = { From 74844697479e64ecb32a8446557136a1b657e03c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 10 Aug 2023 15:11:31 +0200 Subject: [PATCH 3/3] eessi_bot_event_handler.py: improved or removed comments --- eessi_bot_event_handler.py | 66 ++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/eessi_bot_event_handler.py b/eessi_bot_event_handler.py index f993777c..e750f5e5 100644 --- a/eessi_bot_event_handler.py +++ b/eessi_bot_event_handler.py @@ -102,12 +102,10 @@ def handle_issue_comment_event(self, event_info, log_file=None): repo_name = request_body['repository']['full_name'] pr_number = request_body['issue']['number'] - # FIXME add request body text (['comment']['body']) to log message when - # log level is set to debug + # TODO add request body text (['comment']['body']) to log message when + # log level is set to debug self.log(f"Comment in {issue_url} (owned by @{owner}) {action} by @{sender}") - # obtain app name and format for reporting about received & processed - # commands app_name = self.cfg[GITHUB][APP_NAME] command_response_fmt = self.cfg[BOT_CONTROL][COMMAND_RESPONSE_FMT] @@ -118,21 +116,25 @@ def handle_issue_comment_event(self, event_info, log_file=None): # - this serves a double purpose: # 1. check permission # 2. skip any comment updates that were done by the bot itself - # --> thus we prevent the bot from entering an endless loop - # where it reacts on updates to comments it made itself - # NOTE this assumes that the sender of the event is corresponding to - # the bot if the bot updates comments itself and that the bot is not - # given permission in the configuration setting 'command_permission' - # ... in order to prevent surprises we should be careful what the bot - # adds to comments, for example, before updating a comment it could - # run the update through the function checking for a bot command. + # - thus we prevent the bot from entering an endless loop + # where it reacts on updates to comments it made itself + # - this assumes that the sender of an event is corresponding + # to the bot if the bot updates or creates comments itself + # and that the bot is not given permission in the + # configuration setting 'command_permission' + # - in order to prevent surprises we should be very careful + # about what the bot adds to comments, for example, before + # updating a comment it could run the update through the + # function get_bot_command to determine if the comment + # includes a bot command if check_command_permission(sender) is False: self.log(f"account `{sender}` has NO permission to send commands to bot") # need to ensure that the bot is not responding on its own comments # as a quick implementation we check if the sender name contains '[bot]' - # FIXME improve this by querying (and caching) information about the sender of - # an event ALTERNATIVELY we could postpone this test a bit until we - # have parsed the comment and know if it contains any bot command + # TODO improve this by querying (and caching) information about the sender of + # an event + # ALTERNATIVELY we could postpone this test a bit until we + # have parsed the comment and know if it contains any bot command if not sender.endswith('[bot]'): comment_response = f"\n- account `{sender}` has NO permission to send commands to the bot" comment_body = command_response_fmt.format( @@ -152,24 +154,27 @@ def handle_issue_comment_event(self, event_info, log_file=None): comment_received = request_body['comment']['body'] self.log(f"comment action '{action}' is handled") else: - # NOTE we do not report this with a new PR comment, because otherwise any update to a comment would - # let the bot add a response (would be very noisy); we might add a debug mode later + # NOTE we do not respond to an updated PR comment with yet another + # new PR comment, because it would make the bot very noisy or + # worse could result in letting the bot enter an endless loop self.log(f"comment action '{action}' not handled") return # search for commands in comment comment_response = '' commands = [] - # process non-empty (if x) lines (split) in comment + # process any non-empty lines in comment (inner comprehension splits + # comment into lines, outer comprehension ensures only non-empty lines + # are processed further) for line in [x for x in [y.strip() for y in comment_received.split('\n')] if x]: - # FIXME add processed line(s) to log when log level is set to debug + # TODO add processed line(s) to log when log level is set to debug bot_command = get_bot_command(line) if bot_command: try: ebc = EESSIBotCommand(bot_command) except EESSIBotCommandError as bce: self.log(f"ERROR: parsing bot command '{bot_command}' failed with {bce.args}") - # FIXME possibly add more information to log when log level is set to debug + # TODO possibly add more information to log when log level is set to debug comment_response += f"\n- parsing the bot command `{bot_command}`, received" comment_response += f" from sender `{sender}`, failed" continue @@ -178,7 +183,7 @@ def handle_issue_comment_event(self, event_info, log_file=None): comment_response += f"\n- received bot command `{bot_command}`" comment_response += f" from `{sender}`" comment_response += f"\n - expanded format: `{ebc.to_string()}`" - # FIXME add an else branch that logs information for comments not + # TODO add an else branch that logs information for comments not # including a bot command; the logging should only be done when log # level is set to debug @@ -190,10 +195,11 @@ def handle_issue_comment_event(self, event_info, log_file=None): self.log(f"comment response: '{comment_response}'") if not any(map(get_bot_command, comment_response.split('\n'))): - # the 'not any()' ensures that the response would not be considered a bot command itself - # ... together with checking the sender of a comment update this aims - # at preventing the bot to enter an endless loop in commenting on its own - # comments + # the 'not any()' ensures that the response would not be considered + # a bot command itself + # this, together with checking the sender of a comment update, aims + # at preventing the bot to enter an endless loop in commenting on + # its own comments comment_body = command_response_fmt.format( app_name=app_name, comment_response=comment_response, @@ -202,7 +208,7 @@ def handle_issue_comment_event(self, event_info, log_file=None): issue_comment = create_comment(repo_name, pr_number, comment_body) else: self.log(f"update '{comment_response}' is considered to contain bot command ... not creating PR comment") - # FIXME we may want to report this back to the PR on GitHub, e.g., + # TODO we may want to report this back to the PR on GitHub, e.g., # "Oops response message seems to contain a bot command. It is not # displayed here to prevent the bot from entering an endless loop # of commands. Please, check the logs at the bot instance for more @@ -232,7 +238,8 @@ def handle_issue_comment_event(self, event_info, log_file=None): ) issue_comment.edit(comment_body) raise - # only update PR comment once + # only update PR comment once, that is, a single call to + # issue_comment.edit is made in the entire function comment_body = command_response_fmt.format( app_name=app_name, comment_response=comment_response, @@ -256,7 +263,6 @@ def handle_installation_event(self, event_info, log_file=None): request_body = event_info['raw_request_body'] user = request_body['sender']['login'] action = request_body['action'] - # repo_name = request_body['repositories'][0]['full_name'] # not every action has that attribute self.log("App installation event by user %s with action '%s'", user, action) self.log("installation event handled!") @@ -278,10 +284,6 @@ def handle_pull_request_labeled_event(self, event_info, pr): self.log("Process PR labeled event: PR#%s, label '%s'", pr.number, label) if label == "bot:build": - # # run function to build software stack - # if check_build_permission(pr, event_info): - # # use an empty filter - # submit_build_jobs(pr, event_info, EESSIBotActionFilter("")) msg = "Handling the label 'bot:build' is disabled. Use the command `bot: build [FILTER]*` instead." self.log(msg)