From 7bc0d2328f8d0543e8148f092ef06f205a76f64b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 7 Apr 2023 13:01:17 +0200 Subject: [PATCH 01/41] fixing one test due to changed function definition --- tests/test_task_build.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_task_build.py b/tests/test_task_build.py index 7e0669da..7962e07d 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -20,6 +20,7 @@ from unittest.mock import patch # Third party imports (anything installed into the local Python environment) +from datetime import datetime import pytest # Local application imports (anything from EESSI/eessi-bot-software-layer) @@ -267,18 +268,21 @@ def test_create_pr_comment_succeeds(mocked_github, tmpdir): shutil.copyfile("tests/test_app.cfg", "app.cfg") # creating a PR comment print("CREATING PR COMMENT") - job = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed-up") + ym = datetime.today().strftime('%Y.%m') + pr_number = 1 + job = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed-up", ym, pr_number) + job_id = "123" app_name = "pytest" - pr_number = 1 + repo_name = "EESSI/software-layer" + repo = mocked_github.get_repo(repo_name) + pr = repo.get_pull(pr_number) symlink = "/symlink" - comment_id = create_pr_comment(job, job_id, app_name, pr_number, repo_name, mocked_github, symlink) + comment_id = create_pr_comment(job, job_id, app_name, pr, mocked_github, symlink) assert comment_id == 1 # check if created comment includes jobid? print("VERIFYING PR COMMENT") - repo = mocked_github.get_repo(repo_name) - pr = repo.get_pull(pr_number) comment = get_submitted_job_comment(pr, job_id) assert job_id in comment.body From bb0e6279d45de3c0519d8d04f7bc2a384a8159be Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 7 Apr 2023 13:15:47 +0200 Subject: [PATCH 02/41] add base attribute to MockPullRequest --- tests/test_task_build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_task_build.py b/tests/test_task_build.py index 7962e07d..47d9d5c0 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -180,6 +180,7 @@ def __init__(self, pr_number, create_raises='0', create_exception=Exception, cre self.create_raises = create_raises self.create_exception = create_exception self.create_call_count = 0 + self.base = None def create_issue_comment(self, body): def should_raise_exception(): From 2e4c8d2dbf95cfa7dbfc7fd24579170e4bacd33e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 7 Apr 2023 13:31:37 +0200 Subject: [PATCH 03/41] adding nested data structure used to get repository name --- tests/test_task_build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_task_build.py b/tests/test_task_build.py index 47d9d5c0..3e788454 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -20,6 +20,7 @@ from unittest.mock import patch # Third party imports (anything installed into the local Python environment) +from collections import namedtuple from datetime import datetime import pytest @@ -153,6 +154,8 @@ def get_repo(self, repo_name): repo = self.repos[repo_name] return repo +MockBase = namedtuple('MockBase', ['repo']) +MockRepo = namedtuple('MockRepo', ['full_name']) class MockRepository: def __init__(self, repo_name): @@ -165,6 +168,7 @@ def create_pr(self, pr_number, create_raises='0', create_exception=Exception, cr else: self.pull_requests[pr_number] = MockPullRequest(pr_number, create_raises, CreateIssueCommentException, create_fails) + self.pull_requests[pr_number].base = MockBase(MockRepo(self.repo_name)) return self.pull_requests[pr_number] def get_pull(self, pr_number): From 5a8628a0e4096f0f9e402e1c28a1b5860e0e2443 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 7 Apr 2023 13:35:55 +0200 Subject: [PATCH 04/41] change attribute name for MockPullRequest --- tests/test_task_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_task_build.py b/tests/test_task_build.py index 3e788454..bc9020bc 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -178,7 +178,7 @@ def get_pull(self, pr_number): class MockPullRequest: def __init__(self, pr_number, create_raises='0', create_exception=Exception, create_fails=False): - self.pr_number = pr_number + self.number = pr_number self.issue_comments = [] self.create_fails = create_fails self.create_raises = create_raises From f441fb74919be36530093c038dae42c5121697bd Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 7 Apr 2023 13:58:16 +0200 Subject: [PATCH 05/41] fix more failing tests due to changed function definitions --- tests/test_task_build.py | 81 +++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/tests/test_task_build.py b/tests/test_task_build.py index bc9020bc..2d5e284b 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -302,13 +302,18 @@ def test_create_pr_comment_succeeds_none(mocked_github, tmpdir): shutil.copyfile("tests/test_app.cfg", "app.cfg") # creating a PR comment print("CREATING PR COMMENT") - job = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed-up") + ym = datetime.today().strftime('%Y.%m') + pr_number = 1 + job = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed-up", ym, pr_number) + job_id = "123" app_name = "pytest" - pr_number = 1 + repo_name = "EESSI/software-layer" + repo = mocked_github.get_repo(repo_name) + pr = repo.get_pull(pr_number) symlink = "/symlink" - comment_id = create_pr_comment(job, job_id, app_name, pr_number, repo_name, mocked_github, symlink) + comment_id = create_pr_comment(job, job_id, app_name, pr, mocked_github, symlink) assert comment_id == -1 @@ -322,16 +327,19 @@ def test_create_pr_comment_raises_once_then_succeeds(mocked_github, tmpdir): shutil.copyfile("tests/test_app.cfg", "app.cfg") # creating a PR comment print("CREATING PR COMMENT") - job = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed-up") + ym = datetime.today().strftime('%Y.%m') + pr_number = 1 + job = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed-up", ym, pr_number) + job_id = "123" app_name = "pytest" - pr_number = 1 + repo_name = "EESSI/software-layer" - symlink = "/symlink" - comment_id = create_pr_comment(job, job_id, app_name, pr_number, repo_name, mocked_github, symlink) - assert comment_id == 1 repo = mocked_github.get_repo(repo_name) pr = repo.get_pull(pr_number) + symlink = "/symlink" + comment_id = create_pr_comment(job, job_id, app_name, pr, mocked_github, symlink) + assert comment_id == 1 assert pr.create_call_count == 2 @@ -344,17 +352,20 @@ def test_create_pr_comment_always_raises(mocked_github, tmpdir): shutil.copyfile("tests/test_app.cfg", "app.cfg") # creating a PR comment print("CREATING PR COMMENT") - job = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed-up") + ym = datetime.today().strftime('%Y.%m') + pr_number = 1 + job = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed-up", ym, pr_number) + job_id = "123" app_name = "pytest" - pr_number = 1 + repo_name = "EESSI/software-layer" + repo = mocked_github.get_repo(repo_name) + pr = repo.get_pull(pr_number) symlink = "/symlink" with pytest.raises(Exception) as err: - create_pr_comment(job, job_id, app_name, pr_number, repo_name, mocked_github, symlink) + create_pr_comment(job, job_id, app_name, pr, mocked_github, symlink) assert err.type == CreateIssueCommentException - repo = mocked_github.get_repo(repo_name) - pr = repo.get_pull(pr_number) assert pr.create_call_count == 3 @@ -367,29 +378,39 @@ def test_create_pr_comment_three_raises(mocked_github, tmpdir): shutil.copyfile("tests/test_app.cfg", "app.cfg") # creating a PR comment print("CREATING PR COMMENT") - job = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed-up") + ym = datetime.today().strftime('%Y.%m') + pr_number = 1 + job = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed-up", ym, pr_number) + job_id = "123" app_name = "pytest" - pr_number = 1 + repo_name = "EESSI/software-layer" + repo = mocked_github.get_repo(repo_name) + pr = repo.get_pull(pr_number) symlink = "/symlink" with pytest.raises(Exception) as err: - create_pr_comment(job, job_id, app_name, pr_number, repo_name, mocked_github, symlink) + create_pr_comment(job, job_id, app_name, pr, mocked_github, symlink) assert err.type == CreateIssueCommentException - repo = mocked_github.get_repo(repo_name) - pr = repo.get_pull(pr_number) assert pr.create_call_count == 3 -def test_create_metadata_file(tmpdir): +@pytest.mark.repo_name("test_repo") +@pytest.mark.pr_number(999) +def test_create_metadata_file(mocked_github, tmpdir): """Tests for function create_metadata_file.""" # create some test data - job = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed_up_job") + ym = datetime.today().strftime('%Y.%m') + pr_number = 999 + job = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) + job_id = "123" + repo_name = "test_repo" - pr_number = 999 pr_comment_id = 77 - create_metadata_file(job, job_id, repo_name, pr_number, pr_comment_id) + repo = mocked_github.get_repo(repo_name) + pr = repo.get_pull(pr_number) + create_metadata_file(job, job_id, pr, pr_comment_id) expected_file = f"_bot_job{job_id}.metadata" expected_file_path = os.path.join(tmpdir, expected_file) @@ -406,26 +427,26 @@ def test_create_metadata_file(tmpdir): # use directory that does not exist dir_does_not_exist = os.path.join(tmpdir, "dir_does_not_exist") - job2 = Job(dir_does_not_exist, "test/architecture", "EESSI-pilot", "--speed_up_job") + job2 = Job(dir_does_not_exist, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) job_id2 = "222" with pytest.raises(FileNotFoundError): - create_metadata_file(job2, job_id2, repo_name, pr_number, pr_comment_id) + create_metadata_file(job2, job_id2, pr, pr_comment_id) # use directory without write permission dir_without_write_perm = os.path.join("/") - job3 = Job(dir_without_write_perm, "test/architecture", "EESSI-pilot", "--speed_up_job") + job3 = Job(dir_without_write_perm, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) job_id3 = "333" with pytest.raises(OSError): - create_metadata_file(job3, job_id3, repo_name, pr_number, pr_comment_id) + create_metadata_file(job3, job_id3, pr, pr_comment_id) # disk quota exceeded (difficult to create and unlikely to happen because # partition where file is stored is usually very large) # use undefined values for parameters # job_id = None - job4 = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed_up_job") + job4 = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) job_id4 = None - create_metadata_file(job4, job_id4, repo_name, pr_number, pr_comment_id) + create_metadata_file(job4, job_id4, pr, pr_comment_id) expected_file4 = f"_bot_job{job_id}.metadata" expected_file_path4 = os.path.join(tmpdir, expected_file4) @@ -438,7 +459,7 @@ def test_create_metadata_file(tmpdir): # use undefined values for parameters # job.working_dir = None - job5 = Job(None, "test/architecture", "EESSI-pilot", "--speed_up_job") + job5 = Job(None, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) job_id5 = "555" with pytest.raises(TypeError): - create_metadata_file(job5, job_id5, repo_name, pr_number, pr_comment_id) + create_metadata_file(job5, job_id5, pr, pr_comment_id) From 031ae8817e79ef727c9ab8e6581d3d37880ecd4a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 7 Apr 2023 14:00:09 +0200 Subject: [PATCH 06/41] fix flake8 issues --- tests/test_task_build.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_task_build.py b/tests/test_task_build.py index 2d5e284b..4f8fe75a 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -154,9 +154,13 @@ def get_repo(self, repo_name): repo = self.repos[repo_name] return repo + MockBase = namedtuple('MockBase', ['repo']) + + MockRepo = namedtuple('MockRepo', ['full_name']) + class MockRepository: def __init__(self, repo_name): self.repo_name = repo_name From 467bc5b36100781a6164f51e2bab259382f4e1cf Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 7 Apr 2023 15:03:49 +0200 Subject: [PATCH 07/41] add new settings to app.cfg.example --- app.cfg.example | 60 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/app.cfg.example b/app.cfg.example index 5e96cdea..a7c762f8 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -24,6 +24,13 @@ installation_id = 12345678 private_key = PATH_TO_PRIVATE_KEY +[bot_control] +# which GH accounts have the permission to send commands to the bot +# if value is left/empty everyone can send commands +# value can be a space delimited list of GH accounts +command_permission = + + [buildenv] # name of the job script used for building an EESSI stack build_job_script = PATH_TO_EESSI_BOT/scripts/bot-build.slurm @@ -35,13 +42,13 @@ container_cachedir = PATH_TO_SHARED_DIRECTORY # it may happen that we need to customize some CVMFS configuration # the value of cvmfs_customizations is a dictionary which maps a file # name to an entry that needs to be added to that file -cvmfs_customizations = { "/etc/cvmfs/default.local": "CVMFS_HTTP_PROXY=\"http://PROXY_DNS_NAME:3128|http://PROXY_IP_ADDRESS:3128\"" } +# cvmfs_customizations = { "/etc/cvmfs/default.local": "CVMFS_HTTP_PROXY=\"http://PROXY_DNS_NAME:3128|http://PROXY_IP_ADDRESS:3128\"" } # if compute nodes have no internet connection, we need to set http(s)_proxy # or commands such as pip3 cannot download software from package repositories # for example, the temporary EasyBuild is installed via pip3 first -http_proxy = http://PROXY_DNS:3128/ -https_proxy = http://PROXY_DNS:3128/ +# http_proxy = http://PROXY_DNS:3128/ +# https_proxy = http://PROXY_DNS:3128/ # directory under which the bot prepares directories per job # structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR @@ -75,9 +82,12 @@ submit_command = /usr/bin/sbatch # the label 'bot:build' (apparently this cannot be restricted on GitHub) # if value is left/empty everyone can trigger the build # value can be a space delimited list of GH accounts -build_permission = Hafsa-Naeem +build_permission = + +# template for comment when user who set a label has no permission to trigger build jobs no_build_permission_comment = Label `bot:build` has been set by user `{build_labeler}`, but only users `{build_permission_users}` have permission to trigger the action + [deploycfg] # script for uploading built software packages tarball_upload_script = PATH_TO_EESSI_BOT/scripts/eessi-upload-to-staging @@ -108,7 +118,9 @@ upload_policy = once # the label 'bot:deploy' (apparently this cannot be restricted on GitHub) # if value is left/empty everyone can trigger the deployment # value can be a space delimited list of GH accounts -deploy_permission = trz42 +deploy_permission = + +# template for comment when user who set a label has no permission to trigger deploying tarballs no_deploy_permission_comment = Label `bot:deploy` has been set by user `{deploy_labeler}`, but only users `{deploy_permission_users}` have permission to trigger the action @@ -118,6 +130,16 @@ no_deploy_permission_comment = Label `bot:deploy` has been set by user `{deploy_ arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.2xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.2xlarge" } +[repo_targets] +# defines for which repository a arch_target should be build for +# +# EESSI/2021.12 and NESSI/2022.11 +repo_target_map = { "linux/x86_64/amd/zen2" : ["eessi-2021.12","nessi.no-2022.11"] } + +# points to definition of repositories (default EESSI-pilot defined by build container) +repos_cfg_dir = PATH_TO_SHARED_DIRECTORY/cfg_bundles + + # configuration for event handler which receives events from a GitHub repository. [event_handler] # path to the log file to log messages for event handler @@ -141,23 +163,27 @@ poll_interval = 60 # full path to the command for manipulating existing jobs scontrol_command = /usr/bin/scontrol + # variable 'comment' under 'submitted_job_comments' should not be changed as there are regular expression patterns matching it [submitted_job_comments] -initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}` -awaits_release = job id `{job_id}` awaits release by job manager +initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}` +awaits_release = job id `{job_id}` awaits release by job manager + [new_job_comments] -awaits_lauch = job awaits launch by Slurm scheduler +awaits_lauch = job awaits launch by Slurm scheduler + [running_job_comments] -running_job = job `{job_id}` is running +running_job = job `{job_id}` is running + [finished_job_comments] -success = :grin: SUCCESS tarball `{tarball_name}` ({tarball_size} GiB) in job dir -failure = :cry: FAILURE -no_slurm_out = No slurm output `{slurm_out}` in job dir -slurm_out = Found slurm output `{slurm_out}` in job dir -missing_modules = Slurm output lacks message "No missing modules!". -no_tarball_message = Slurm output lacks message about created tarball. -no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. -multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. \ No newline at end of file +success = :grin: SUCCESS tarball `{tarball_name}` ({tarball_size} GiB) in job dir +failure = :cry: FAILURE +no_slurm_out = No slurm output `{slurm_out}` in job dir +slurm_out = Found slurm output `{slurm_out}` in job dir +missing_modules = Slurm output lacks message "No missing modules!". +no_tarball_message = Slurm output lacks message about created tarball. +no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. +multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. From 21c79b82f7a14686b81d3ebcc9c519969ac1e6c8 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 8 Apr 2023 19:54:47 +0200 Subject: [PATCH 08/41] move job metadata functions to separate module --- tools/job_metadata.py | 72 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tools/job_metadata.py diff --git a/tools/job_metadata.py b/tools/job_metadata.py new file mode 100644 index 00000000..7e20ccb5 --- /dev/null +++ b/tools/job_metadata.py @@ -0,0 +1,72 @@ +# This file is part of the EESSI build-and-deploy bot, +# see https://github.com/EESSI/eessi-bot-software-layer +# +# The bot helps with requests to add software installations to the +# EESSI software layer, see https://github.com/EESSI/software-layer +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# +from collections import namedtuple +import configparser +import os +import sys + +from pyghee.utils import log, error +from tasks.build import Job + + +def create_metadata_file(job, job_id, pr, pr_comment_id): + """Create metadata file in submission dir. + + Args: + job (named tuple): key data about job that has been submitted + job_id (string): id of submitted job + pr (github.PullRequest.Pullrequest): object to interact with pull request + pr_comment_id (int): id of PR comment + """ + fn = sys._getframe().f_code.co_name + + repo_name = pr.base.repo.full_name + + # create _bot_job.metadata file in submission directory + bot_jobfile = configparser.ConfigParser() + bot_jobfile['PR'] = {'repo': repo_name, 'pr_number': pr.number, 'pr_comment_id': pr_comment_id} + bot_jobfile_path = os.path.join(job.working_dir, f'_bot_job{job_id}.metadata') + with open(bot_jobfile_path, 'w') as bjf: + bot_jobfile.write(bjf) + log(f"{fn}(): created job metadata file {bot_jobfile_path}") + + +def read_metadata_file(job_metadata_path, log_file=None): + """ + Try to read metadata file and return its PR section. Return None in + case of failure (treat all cases as if the file did not exist): + - file does not exist, + - file exists but does not contain PR section, + - file exists but parsing/reading resulted in an exception. + + Args: + job_metadata_path (string): path to job metadata file + log_file (string): path to log file + """ + # check if metadata file exist + if os.path.isfile(job_metadata_path): + log(f"Found metadata file at {job_metadata_path}", log_file) + metadata = configparser.ConfigParser() + try: + metadata.read(job_metadata_path) + except Exception as err: + # error would let the process exist, this is too harsh, + log(f"Unable to read job metadata file {job_metadata_path}: {err}") + return None + + # get PR section + if "PR" in metadata: + return metadata["PR"] + else: + return None + else: + log(f"No metadata file found at {job_metadata_path}, so not a bot job", log_file) + return None From b0096253aa5c6ad9ce888a4bf03c828ed4a785e5 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 8 Apr 2023 20:02:14 +0200 Subject: [PATCH 09/41] add step to check result in job script --- scripts/bot-build.slurm | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/bot-build.slurm b/scripts/bot-build.slurm index 93f0ddc0..a08cb459 100755 --- a/scripts/bot-build.slurm +++ b/scripts/bot-build.slurm @@ -29,3 +29,16 @@ if [ -f bot/build.sh ]; then else fatal_error "could not find bot/build.sh script in '${PWD}'" fi +echo "bot/build.sh finished" +if [ -f bot/check-result.sh ]; then + echo "bot/check-result.sh script found in '${PWD}', so running it!" + bot/check-result.sh +else + echo "could not find bot/check-result.sh script in '${PWD}' ..." + echo "... depositing default _bot_job${SLURM_JOB_ID}.result file in '${PWD}'" + cat > _bot_job${SLURM_JOB_ID}.result < Date: Sat, 8 Apr 2023 20:03:20 +0200 Subject: [PATCH 10/41] move metadata function to separate module --- tasks/build.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/tasks/build.py b/tasks/build.py index f4a2bc33..9dd09874 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -23,6 +23,7 @@ from pyghee.utils import log, error from retry.api import retry_call from tools import config, run_cmd +from tools.job_metadata import create_metadata_file APP_NAME = "app_name" AWAITS_RELEASE = "awaits_release" @@ -572,28 +573,6 @@ def submit_job(job, cfg): return job_id, symlink -def create_metadata_file(job, job_id, pr, pr_comment_id): - """Create metadata file in submission dir. - - Args: - job (named tuple): key data about job that has been submitted - job_id (string): id of submitted job - pr (github.PullRequest.Pullrequest): object to interact with pull request - pr_comment_id (int): id of PR comment - """ - fn = sys._getframe().f_code.co_name - - repo_name = pr.base.repo.full_name - - # create _bot_job.metadata file in submission directory - bot_jobfile = configparser.ConfigParser() - bot_jobfile['PR'] = {'repo': repo_name, 'pr_number': pr.number, 'pr_comment_id': pr_comment_id} - bot_jobfile_path = os.path.join(job.working_dir, f'_bot_job{job_id}.metadata') - with open(bot_jobfile_path, 'w') as bjf: - bot_jobfile.write(bjf) - log(f"{fn}(): created job metadata file {bot_jobfile_path}") - - def create_pr_comment(job, job_id, app_name, pr, gh, symlink): """create pr comment for newly submitted job From 936fef496af2954ccbdd0101745d598dd58cd7b6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 8 Apr 2023 20:04:50 +0200 Subject: [PATCH 11/41] move metadata function to own module + sketch procedure to check job result --- eessi_bot_job_manager.py | 55 +++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 940162f0..0f91b42c 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -40,6 +40,7 @@ from datetime import datetime, timezone from tools import config, run_cmd from tools.pr_comments import get_submitted_job_comment, update_comment +from tools.job_metadata import read_metadata_file from pyghee.utils import log, error @@ -194,24 +195,8 @@ def read_job_pr_metadata(self, job_metadata_path): """ Check if metadata file exists, read it and return 'PR' section if so, return None if not. """ - # check if metadata file exist - if os.path.isfile(job_metadata_path): - log(f"Found metadata file at {job_metadata_path}", self.logfile) - metadata = configparser.ConfigParser() - try: - metadata.read(job_metadata_path) - except Exception as err: - error(f"Unable to read job metadata file {job_metadata_path}: {err}") - - # get PR section - if "PR" in metadata: - metadata_pr = metadata["PR"] - else: - metadata_pr = {} - return metadata_pr - else: - log(f"No metadata file found at {job_metadata_path}, so not a bot job", self.logfile) - return None + # just use a function provided by module tools.job_metadata + return read_metadata_file(job_metadata_path, self.logfile) # job_manager.process_new_job(current_jobs[nj]) def process_new_job(self, new_job): @@ -402,6 +387,40 @@ def process_running_jobs(self, running_job): # job_manager.process_finished_job(known_jobs[fj]) def process_finished_job(self, finished_job): + # procedure: + # 1. check if file _bot_jobJOBID.result exists --> if so read it and + # prepare update to PR comment + # result file contents: + # result={SUCCESS|FAILURE|UNKNOWN} + # msg=multiline string that is put into details element + # built_artefacts=multiline string with (relative) path names + # jobid= + # runtime= + # resources_requested=CPU:x,RAM:yG,DISK:zG + # resources_allocated=CPU:x,RAM:yG,DISK:zG + # resources_used=CPU:x,RAM:yG,DISK:zG + # 2. if file doesn't exist, use the below procedure (only until target + # repositories are updated to create such a result file) + + # 1. check if _bot_jobJOBID.result exits + job_dir = os.path.join(self.submitted_jobs_dir, finished_job["jobid"]) + job_result_file = f"_bot_job{finished_job['jobid']}.result" + job_result_file_path = os.path.join(job_dir, job_result_file) + job_results = self.read_job_result(job_result_file_path) + if job_results: + # TODO process results & return + + # we should only gotten here if there was no job result file or it could + # not be read. if the old code has been moved to the target repository we + # need to add a standard message ala "UNKNOWN result because no job + # result file found" + + # NOTE if also the deploy functionality is changed such to use the + # results file the bot really becomes independent of what it builds + + # TODO the below should be done by the target repository's script + # bot/check-result.sh which should produce a file + # _bot_jobJOBID.result # check result # ("No missing packages!", "eessi-.*.tar.gz") # TODO as is, this requires knowledge about the build process. From 526626d06c3a1f61581a91f071d5b0564c987082 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 8 Apr 2023 20:21:21 +0200 Subject: [PATCH 12/41] make read metadata function more generic --- eessi_bot_job_manager.py | 6 +++++- tools/job_metadata.py | 24 ++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 0f91b42c..ea746475 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -196,7 +196,11 @@ def read_job_pr_metadata(self, job_metadata_path): Check if metadata file exists, read it and return 'PR' section if so, return None if not. """ # just use a function provided by module tools.job_metadata - return read_metadata_file(job_metadata_path, self.logfile) + metadata = read_metadata_file(job_metadata_path, self.logfile) + if metadata and "PR" in metadata: + return metadata["PR"] + else: + return None # job_manager.process_new_job(current_jobs[nj]) def process_new_job(self, new_job): diff --git a/tools/job_metadata.py b/tools/job_metadata.py index 7e20ccb5..ed7fd3ca 100644 --- a/tools/job_metadata.py +++ b/tools/job_metadata.py @@ -39,34 +39,30 @@ def create_metadata_file(job, job_id, pr, pr_comment_id): log(f"{fn}(): created job metadata file {bot_jobfile_path}") -def read_metadata_file(job_metadata_path, log_file=None): +def read_metadata_file(metadata_path, log_file=None): """ - Try to read metadata file and return its PR section. Return None in + Try to read metadata file and return it. Return None in case of failure (treat all cases as if the file did not exist): - file does not exist, - - file exists but does not contain PR section, - file exists but parsing/reading resulted in an exception. Args: - job_metadata_path (string): path to job metadata file + metadata_path (string): path to metadata file log_file (string): path to log file """ # check if metadata file exist - if os.path.isfile(job_metadata_path): - log(f"Found metadata file at {job_metadata_path}", log_file) + if os.path.isfile(metadata_path): + log(f"Found metadata file at {metadata_path}", log_file) metadata = configparser.ConfigParser() try: - metadata.read(job_metadata_path) + metadata.read(metadata_path) except Exception as err: # error would let the process exist, this is too harsh, - log(f"Unable to read job metadata file {job_metadata_path}: {err}") + # we return None and let the caller decide what to do. + log(f"Unable to read metadata file {metadata_path}: {err}") return None - # get PR section - if "PR" in metadata: - return metadata["PR"] - else: - return None + return metadata else: - log(f"No metadata file found at {job_metadata_path}, so not a bot job", log_file) + log(f"No metadata file found at {metadata_path}.", log_file) return None From 50d9f4b12c6e9fca4abad5174c4bfb9d6868756f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 8 Apr 2023 20:29:40 +0200 Subject: [PATCH 13/41] function to read job result + change in result file format --- eessi_bot_job_manager.py | 16 ++++++++++++++-- scripts/bot-build.slurm | 5 +++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index ea746475..0a8d287f 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -202,6 +202,17 @@ def read_job_pr_metadata(self, job_metadata_path): else: return None + def read_job_result(self, job_result_file_path): + """ + Check if result file exists, read it and return 'RESULT section if so, return None if not. + """ + # just use a function provided by module tools.job_metadata + result = read_metadata_file(job_result_file_path, self.logfile) + if result and "RESULT" in result: + return result["RESULT"] + else: + return None + # job_manager.process_new_job(current_jobs[nj]) def process_new_job(self, new_job): # create symlink in submitted_jobs_dir (destination is the working @@ -395,8 +406,9 @@ def process_finished_job(self, finished_job): # 1. check if file _bot_jobJOBID.result exists --> if so read it and # prepare update to PR comment # result file contents: - # result={SUCCESS|FAILURE|UNKNOWN} - # msg=multiline string that is put into details element + # [RESULT] + # summary={SUCCESS|FAILURE|UNKNOWN} + # details=multiline string that is put into details element # built_artefacts=multiline string with (relative) path names # jobid= # runtime= diff --git a/scripts/bot-build.slurm b/scripts/bot-build.slurm index a08cb459..13263b7c 100755 --- a/scripts/bot-build.slurm +++ b/scripts/bot-build.slurm @@ -37,8 +37,9 @@ else echo "could not find bot/check-result.sh script in '${PWD}' ..." echo "... depositing default _bot_job${SLURM_JOB_ID}.result file in '${PWD}'" cat > _bot_job${SLURM_JOB_ID}.result < Date: Sat, 8 Apr 2023 21:23:45 +0200 Subject: [PATCH 14/41] first sketch of using job result --- eessi_bot_job_manager.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 0a8d287f..2d211126 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -425,7 +425,22 @@ def process_finished_job(self, finished_job): job_results = self.read_job_result(job_result_file_path) if job_results: # TODO process results & return + # get summary + summary = job_results.get(JOB_RESULT_SUMMARY, ":shrug: UNKOWN") + # get details + details = job_results.get(JOB_RESULT_DETAILS, "* _no details provided_") + # get built_artefacts + built_artefacts = job_results.get(JOB_RESULT_ARTEFACTS, "* _no built artefacts reported_") + dt = datetime.now(timezone.utc) + + job_result_comment_fmt = config.read_config()[JOB_RESULT_COMMENT_FMT] + comment_update = f"\n|{dt.strftime('%b %d %X %Z %Y')}|finished|" + comment_update += job_result_comment_fmt.format( + summary=summary, + details=details, + built_artefacts=built_artefacts + ) # we should only gotten here if there was no job result file or it could # not be read. if the old code has been moved to the target repository we # need to add a standard message ala "UNKNOWN result because no job From 141c719dbb3760a75297a0f55b8a4815461b9ec7 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 9 Apr 2023 06:45:36 +0200 Subject: [PATCH 15/41] introduce namedtuple PRComment + use it for create_metadata_file + tests --- eessi_bot_job_manager.py | 21 +++++++++++++-------- tasks/build.py | 4 +++- tests/test_task_build.py | 19 ++++++++++--------- tools/job_metadata.py | 16 ++++++++++------ tools/pr_comments.py | 4 ++++ 5 files changed, 40 insertions(+), 24 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 2d211126..e80e94d4 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -441,13 +441,18 @@ def process_finished_job(self, finished_job): details=details, built_artefacts=built_artefacts ) - # we should only gotten here if there was no job result file or it could - # not be read. if the old code has been moved to the target repository we - # need to add a standard message ala "UNKNOWN result because no job - # result file found" + + # establish contact to pull request on github + gh = github.get_instance() + + # we should only gotten here if there was no job result file or it + # could not be read. if the old code has been moved to the target + # repository we need to add a standard message ala "UNKNOWN result + # because no job result file found" # NOTE if also the deploy functionality is changed such to use the - # results file the bot really becomes independent of what it builds + # results file the bot really becomes independent of what it + # builds # TODO the below should be done by the target repository's script # bot/check-result.sh which should produce a file @@ -471,15 +476,15 @@ def process_finished_job(self, finished_job): job_dir = os.path.join(self.submitted_jobs_dir, finished_job["jobid"]) sym_dst = os.readlink(job_dir) - # TODO create function for obtaining values from metadata file - # might be based on allowing multiple configuration files - # in tools/config.py + # read some information from job metadata file metadata_file = "_bot_job%s.metadata" % finished_job["jobid"] job_metadata_path = os.path.join(job_dir, metadata_file) # check if metadata file exist metadata_pr = self.read_job_pr_metadata(job_metadata_path) if metadata_pr is None: + # TODO should we raise the Exception here? maybe first process + # the finished job and raise an exception at the end? raise Exception("Unable to find metadata file") # get repo name diff --git a/tasks/build.py b/tasks/build.py index 9dd09874..6c4e5301 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -655,8 +655,10 @@ def submit_build_jobs(pr, event_info, action_filter): # report submitted job pr_comment_id = create_pr_comment(job, job_id, app_name, pr, gh, symlink) + pr_comment = PRComment(pr.base.repo.full_name, pr.number, pr_comment_id) + # create _bot_job.metadata file in submission directory - create_metadata_file(job, job_id, pr, pr_comment_id) + create_metadata_file(job, job_id, pr_comment) return_msg = f"created jobs: {', '.join(job_ids)}" return return_msg diff --git a/tests/test_task_build.py b/tests/test_task_build.py index 4f8fe75a..9a9f3923 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -25,9 +25,10 @@ import pytest # Local application imports (anything from EESSI/eessi-bot-software-layer) -from tasks.build import Job, create_metadata_file, create_pr_comment +from tasks.build import Job, create_pr_comment from tools import run_cmd, run_subprocess -from tools.pr_comments import get_submitted_job_comment +from tools.pr_comments import get_submitted_job_comment, PRComment +from tools.job_metadata import create_metadata_file # Local tests imports (reusing code from other tests) from tests.test_tools_pr_comments import MockIssueComment @@ -412,9 +413,9 @@ def test_create_metadata_file(mocked_github, tmpdir): repo_name = "test_repo" pr_comment_id = 77 - repo = mocked_github.get_repo(repo_name) - pr = repo.get_pull(pr_number) - create_metadata_file(job, job_id, pr, pr_comment_id) + pr_comment = PRComment(repo_name, pr_number, pr_comment_id) + + create_metadata_file(job, job_id, pr_comment) expected_file = f"_bot_job{job_id}.metadata" expected_file_path = os.path.join(tmpdir, expected_file) @@ -434,14 +435,14 @@ def test_create_metadata_file(mocked_github, tmpdir): job2 = Job(dir_does_not_exist, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) job_id2 = "222" with pytest.raises(FileNotFoundError): - create_metadata_file(job2, job_id2, pr, pr_comment_id) + create_metadata_file(job2, job_id2, pr_comment) # use directory without write permission dir_without_write_perm = os.path.join("/") job3 = Job(dir_without_write_perm, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) job_id3 = "333" with pytest.raises(OSError): - create_metadata_file(job3, job_id3, pr, pr_comment_id) + create_metadata_file(job3, job_id3, pr_comment) # disk quota exceeded (difficult to create and unlikely to happen because # partition where file is stored is usually very large) @@ -450,7 +451,7 @@ def test_create_metadata_file(mocked_github, tmpdir): # job_id = None job4 = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) job_id4 = None - create_metadata_file(job4, job_id4, pr, pr_comment_id) + create_metadata_file(job4, job_id4, pr_comment) expected_file4 = f"_bot_job{job_id}.metadata" expected_file_path4 = os.path.join(tmpdir, expected_file4) @@ -466,4 +467,4 @@ def test_create_metadata_file(mocked_github, tmpdir): job5 = Job(None, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) job_id5 = "555" with pytest.raises(TypeError): - create_metadata_file(job5, job_id5, pr, pr_comment_id) + create_metadata_file(job5, job_id5, pr_comment) diff --git a/tools/job_metadata.py b/tools/job_metadata.py index ed7fd3ca..da12fee4 100644 --- a/tools/job_metadata.py +++ b/tools/job_metadata.py @@ -8,31 +8,35 @@ # # license: GPLv2 # -from collections import namedtuple +# from collections import namedtuple import configparser import os import sys from pyghee.utils import log, error from tasks.build import Job +from tools.pr_comments import PRComment -def create_metadata_file(job, job_id, pr, pr_comment_id): +def create_metadata_file(job, job_id, pr_comment): """Create metadata file in submission dir. Args: job (named tuple): key data about job that has been submitted job_id (string): id of submitted job - pr (github.PullRequest.Pullrequest): object to interact with pull request - pr_comment_id (int): id of PR comment + pr_comment (PRComment): contains repo_name, pr_number and pr_comment_id """ fn = sys._getframe().f_code.co_name - repo_name = pr.base.repo.full_name + repo_name = pr_comment.repo_name + pr_number = pr_comment.pr_number + pr_comment_id = pr_comment.pr_comment_id # create _bot_job.metadata file in submission directory bot_jobfile = configparser.ConfigParser() - bot_jobfile['PR'] = {'repo': repo_name, 'pr_number': pr.number, 'pr_comment_id': pr_comment_id} + bot_jobfile['PR'] = {'repo': repo_name, + 'pr_number': pr_number, + 'pr_comment_id': pr_comment_id} bot_jobfile_path = os.path.join(job.working_dir, f'_bot_job{job_id}.metadata') with open(bot_jobfile_path, 'w') as bjf: bot_jobfile.write(bjf) diff --git a/tools/pr_comments.py b/tools/pr_comments.py index 16e43283..956e990c 100644 --- a/tools/pr_comments.py +++ b/tools/pr_comments.py @@ -13,12 +13,16 @@ # import re +from collections import namedtuple from connections import github from pyghee.utils import log from retry import retry from retry.api import retry_call +PRComment = namedtuple('PRComment', ('repo_name', 'pr_number', 'pr_comment_id')) + + @retry(Exception, tries=5, delay=1, backoff=2, max_delay=30) def get_comment(pr, search_pattern): """get comment using the search pattern From 487dddee4ed0a049a04c57e29cda58a7a96933cd Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 9 Apr 2023 07:31:50 +0200 Subject: [PATCH 16/41] format details as list items + update comment + polishing --- eessi_bot_job_manager.py | 66 ++++++++++++++++++++++++++++------------ tools/pr_comments.py | 15 +++++++++ 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index e80e94d4..7004ea98 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -33,13 +33,14 @@ import os import re import time +import sys from connections import github from tools.args import job_manager_parse from datetime import datetime, timezone from tools import config, run_cmd -from tools.pr_comments import get_submitted_job_comment, update_comment +from tools.pr_comments import get_submitted_job_comment, update_comment, make_html_list_items from tools.job_metadata import read_metadata_file from pyghee.utils import log, error @@ -402,6 +403,7 @@ def process_running_jobs(self, running_job): # job_manager.process_finished_job(known_jobs[fj]) def process_finished_job(self, finished_job): + fn = sys._getframe().f_code.co_name # procedure: # 1. check if file _bot_jobJOBID.result exists --> if so read it and # prepare update to PR comment @@ -428,9 +430,11 @@ def process_finished_job(self, finished_job): # get summary summary = job_results.get(JOB_RESULT_SUMMARY, ":shrug: UNKOWN") # get details - details = job_results.get(JOB_RESULT_DETAILS, "* _no details provided_") + details = job_results.get(JOB_RESULT_DETAILS, "_no details provided_") + details_list = make_html_list_items(details) # get built_artefacts - built_artefacts = job_results.get(JOB_RESULT_ARTEFACTS, "* _no built artefacts reported_") + built_artefacts = job_results.get(JOB_RESULT_ARTEFACTS, "_no built artefacts reported_") + built_artefacts_list = make_html_list_items(built_artefacts) dt = datetime.now(timezone.utc) @@ -438,13 +442,47 @@ def process_finished_job(self, finished_job): comment_update = f"\n|{dt.strftime('%b %d %X %Z %Y')}|finished|" comment_update += job_result_comment_fmt.format( summary=summary, - details=details, - built_artefacts=built_artefacts + details=details_list, + built_artefacts=built_artefacts_list ) + # obtain id of PR comment to be updated (from _bot_jobID.metadata) + metadata_file = f"_bot_job{finished_job['jobid']}.metadata" + job_metadata_path = os.path.join(job_dir, metadata_file) + metadata_pr = self.read_job_pr_metadata(job_metadata_path) + if metadata_pr is None: + # TODO should we raise the Exception here? maybe first process + # the finished job and raise an exception at the end? + raise Exception("Unable to find metadata file") + + # get repo name + repo_name = metadata_pr.get("repo", None) + # get pr number + pr_number = metadata_pr.get("pr_number", -1) + # get pr comment id + pr_comment_id = metadata_pr.get("pr_comment_id", -1) + log(f"{fn}(): pr comment id {pr_comment_id}", self.logfile) + # establish contact to pull request on github gh = github.get_instance() + repo = gh.get_repo(repo_name) + pull_request = repo.get_pull(int(pr_number)) + + update_comment(pr_comment_id, pull_request, comment_update) + + # move symlink from job_ids_dir/submitted to jobs_ids_dir/finished + old_symlink = os.path.join( + self.submitted_jobs_dir, finished_job["jobid"]) + finished_jobs_dir = os.path.join(self.job_ids_dir, "finished") + os.makedirs(finished_jobs_dir, exist_ok=True) + new_symlink = os.path.join( + finished_jobs_dir, finished_job["jobid"]) + log(f"{fn}(): os.rename({old_symlink},{new_symlink})", self.logfile) + os.rename(old_symlink, new_symlink) + + return foo + # we should only gotten here if there was no job result file or it # could not be read. if the old code has been moved to the target # repository we need to add a standard message ala "UNKNOWN result @@ -500,11 +538,7 @@ def process_finished_job(self, finished_job): finished_job_cmnt = get_submitted_job_comment(pull_request, finished_job['jobid']) if finished_job_cmnt: - log( - "process_finished_job(): found comment with id %s" - % finished_job_cmnt.id, - self.logfile, - ) + log(f"{fn}(): found comment with id {finished_job_cmnt.id}", self.logfile) finished_job["comment_id"] = finished_job_cmnt.id # analyse job result @@ -611,11 +645,8 @@ def process_finished_job(self, finished_job): if "comment_id" in finished_job: update_comment(finished_job["comment_id"], pull_request, comment_update) else: - log( - "process_finished_job(): did not obtain/find a " - "comment for job '%s'" % finished_job["jobid"], - self.logfile, - ) + job_id = finished_job["jobid"] + log(f"{fn}(): did not find a comment for job {job_id}", self.logfile) # TODO just create one? # move symlink from job_ids_dir/submitted to jobs_ids_dir/finished @@ -625,10 +656,7 @@ def process_finished_job(self, finished_job): os.makedirs(finished_jobs_dir, exist_ok=True) new_symlink = os.path.join( finished_jobs_dir, finished_job["jobid"]) - log( - f"process_finished_job(): os.rename({old_symlink},{new_symlink})", - self.logfile, - ) + log(f"{fn}(): os.rename({old_symlink},{new_symlink})", self.logfile) os.rename(old_symlink, new_symlink) diff --git a/tools/pr_comments.py b/tools/pr_comments.py index 956e990c..adcd1b25 100644 --- a/tools/pr_comments.py +++ b/tools/pr_comments.py @@ -103,3 +103,18 @@ def update_pr_comment(event_info, update): pull_request = repo.get_pull(pr_number) issue_comment = pull_request.get_issue_comment(issue_id) issue_comment.edit(comment_new + update) + + +def make_html_list_items(lines): + """Makes HTML list from lines. + + Args: + lines (string): multiline string + + Returns: + multiline (string): formatted as HTML list items + """ + html_list_items = "" + for line in lines.split("\n"): + html_list_items += f"
  • {line}
    • " + return html_list_items From be5445b59cdbd9d9692a0807128a14d1c0062956 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 9 Apr 2023 08:25:46 +0200 Subject: [PATCH 17/41] restructure code + provide results template --- app.cfg.example | 4 ++ eessi_bot_job_manager.py | 124 ++++++++++++++++++++++----------------- 2 files changed, 73 insertions(+), 55 deletions(-) diff --git a/app.cfg.example b/app.cfg.example index a7c762f8..d1e3c109 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -187,3 +187,7 @@ missing_modules = Slurm output lacks message "No missing modules!". no_tarball_message = Slurm output lacks message about created tarball. no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. +job_result_comment_fmt = +
    {summary} _click triangle for details_ +Details:
      {details}
    +Artefacts:
      {artefacts}
    diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 7004ea98..712cc1e3 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -59,6 +59,11 @@ SLURM_OUT = "slurm_out" SUCCESS = "success" +JOB_RESULT_COMMENT_FMT = "job_result_comment_fmt" +JOB_RESULT_SUMMARY = "summary" +JOB_RESULT_DETAILS = "details" +JOB_RESULT_ARTEFACTS = "artefacts" + REQUIRED_CONFIG = { NEW_JOB_COMMENTS: [AWAITS_LAUCH], RUNNING_JOB_COMMENTS: [RUNNING_JOB], @@ -401,29 +406,52 @@ def process_running_jobs(self, running_job): self.logfile, ) - # job_manager.process_finished_job(known_jobs[fj]) def process_finished_job(self, finished_job): + """Process a finished job (move symlink, log and update PR comment). + + Args: + finished_job (dict): dictionary with information about job + """ fn = sys._getframe().f_code.co_name - # procedure: - # 1. check if file _bot_jobJOBID.result exists --> if so read it and - # prepare update to PR comment - # result file contents: - # [RESULT] - # summary={SUCCESS|FAILURE|UNKNOWN} - # details=multiline string that is put into details element - # built_artefacts=multiline string with (relative) path names - # jobid= - # runtime= - # resources_requested=CPU:x,RAM:yG,DISK:zG - # resources_allocated=CPU:x,RAM:yG,DISK:zG - # resources_used=CPU:x,RAM:yG,DISK:zG - # 2. if file doesn't exist, use the below procedure (only until target - # repositories are updated to create such a result file) - - # 1. check if _bot_jobJOBID.result exits - job_dir = os.path.join(self.submitted_jobs_dir, finished_job["jobid"]) - job_result_file = f"_bot_job{finished_job['jobid']}.result" - job_result_file_path = os.path.join(job_dir, job_result_file) + + # PROCEDURE + # - MOVE symlink to finished dir + # - REPORT status always to log, if accessible also to PR comment + + job_id = finished_job['jobid'] + + # MOVE symlink from job_ids_dir/submitted to jobs_ids_dir/finished + old_symlink = os.path.join(self.submitted_jobs_dir, job_id) + + finished_jobs_dir = os.path.join(self.job_ids_dir, "finished") + os.makedirs(finished_jobs_dir, exist_ok=True) + + new_symlink = os.path.join(finished_jobs_dir, job_id) + + log(f"{fn}(): os.rename({old_symlink},{new_symlink})", self.logfile) + os.rename(old_symlink, new_symlink) + + # REPORT status (to logfile in any case, to PR comment if accessible) + # NEW + # check if file _bot_jobJOBID.result exists --> if so read it and + # prepare update to PR comment + # result file contents: + # [RESULT] + # summary={SUCCESS|FAILURE|UNKNOWN} + # details=multiline string that is put into details element + # built_artefacts=multiline string with (relative) path names + # jobid= + # runtime= + # resources_requested=CPU:x,RAM:yG,DISK:zG + # resources_allocated=CPU:x,RAM:yG,DISK:zG + # resources_used=CPU:x,RAM:yG,DISK:zG + # OLD + # if file doesn't exist, use the below procedure (only until target + # repositories are updated to create such a result file) + + # NEW check if _bot_jobJOBID.result exits + job_result_file = f"_bot_job{job_id}.result" + job_result_file_path = os.path.join(new_symlink, job_result_file) job_results = self.read_job_result(job_result_file_path) if job_results: # TODO process results & return @@ -433,8 +461,16 @@ def process_finished_job(self, finished_job): details = job_results.get(JOB_RESULT_DETAILS, "_no details provided_") details_list = make_html_list_items(details) # get built_artefacts - built_artefacts = job_results.get(JOB_RESULT_ARTEFACTS, "_no built artefacts reported_") - built_artefacts_list = make_html_list_items(built_artefacts) + artefacts = job_results.get(JOB_RESULT_ARTEFACTS, "_no artefacts reported_") + artefacts_list = make_html_list_items(artefacts) + + # TODO report to log + log(f"{fn}(): finished job {job_id}\n" + f"########\n" + f"summary: {summary}\n" + f"details: {details}\n" + f"artefacts: {artefacts}\n" + f"########\n", self.logfile) dt = datetime.now(timezone.utc) @@ -443,17 +479,17 @@ def process_finished_job(self, finished_job): comment_update += job_result_comment_fmt.format( summary=summary, details=details_list, - built_artefacts=built_artefacts_list + artefacts=artefacts_list ) # obtain id of PR comment to be updated (from _bot_jobID.metadata) - metadata_file = f"_bot_job{finished_job['jobid']}.metadata" - job_metadata_path = os.path.join(job_dir, metadata_file) + metadata_file = f"_bot_job{job_id}.metadata" + job_metadata_path = os.path.join(new_symlink, metadata_file) metadata_pr = self.read_job_pr_metadata(job_metadata_path) if metadata_pr is None: # TODO should we raise the Exception here? maybe first process # the finished job and raise an exception at the end? - raise Exception("Unable to find metadata file") + raise Exception("Unable to find metadata file ... skip updating PR comment") # get repo name repo_name = metadata_pr.get("repo", None) @@ -471,17 +507,7 @@ def process_finished_job(self, finished_job): update_comment(pr_comment_id, pull_request, comment_update) - # move symlink from job_ids_dir/submitted to jobs_ids_dir/finished - old_symlink = os.path.join( - self.submitted_jobs_dir, finished_job["jobid"]) - finished_jobs_dir = os.path.join(self.job_ids_dir, "finished") - os.makedirs(finished_jobs_dir, exist_ok=True) - new_symlink = os.path.join( - finished_jobs_dir, finished_job["jobid"]) - log(f"{fn}(): os.rename({old_symlink},{new_symlink})", self.logfile) - os.rename(old_symlink, new_symlink) - - return foo + return # we should only gotten here if there was no job result file or it # could not be read. if the old code has been moved to the target @@ -511,12 +537,11 @@ def process_finished_job(self, finished_job): gh = github.get_instance() # set some variables for accessing work dir of job - job_dir = os.path.join(self.submitted_jobs_dir, finished_job["jobid"]) - sym_dst = os.readlink(job_dir) + sym_dst = os.readlink(new_symlink) # read some information from job metadata file - metadata_file = "_bot_job%s.metadata" % finished_job["jobid"] - job_metadata_path = os.path.join(job_dir, metadata_file) + metadata_file = f"_bot_job{job_id}.metadata" + job_metadata_path = os.path.join(new_symlink, metadata_file) # check if metadata file exist metadata_pr = self.read_job_pr_metadata(job_metadata_path) @@ -535,15 +560,14 @@ def process_finished_job(self, finished_job): # determine comment to be updated if "comment_id" not in finished_job: - finished_job_cmnt = get_submitted_job_comment(pull_request, finished_job['jobid']) + finished_job_cmnt = get_submitted_job_comment(pull_request, job_id) if finished_job_cmnt: log(f"{fn}(): found comment with id {finished_job_cmnt.id}", self.logfile) finished_job["comment_id"] = finished_job_cmnt.id # analyse job result - slurm_out = os.path.join(sym_dst, "slurm-%s.out" % - finished_job["jobid"]) + slurm_out = os.path.join(sym_dst, "slurm-{job_id}.out") # determine all tarballs that are stored in # the job directory (only expecting 1) @@ -649,16 +673,6 @@ def process_finished_job(self, finished_job): log(f"{fn}(): did not find a comment for job {job_id}", self.logfile) # TODO just create one? - # move symlink from job_ids_dir/submitted to jobs_ids_dir/finished - old_symlink = os.path.join( - self.submitted_jobs_dir, finished_job["jobid"]) - finished_jobs_dir = os.path.join(self.job_ids_dir, "finished") - os.makedirs(finished_jobs_dir, exist_ok=True) - new_symlink = os.path.join( - finished_jobs_dir, finished_job["jobid"]) - log(f"{fn}(): os.rename({old_symlink},{new_symlink})", self.logfile) - os.rename(old_symlink, new_symlink) - def main(): """Main function.""" From 367f248e1192fe7d0ff7e4b5c3001c804be10991 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 9 Apr 2023 08:34:15 +0200 Subject: [PATCH 18/41] address flake8 issues --- eessi_bot_job_manager.py | 15 +++++---------- tasks/build.py | 1 + tools/job_metadata.py | 6 +++--- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 712cc1e3..d5d3283b 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -28,7 +28,7 @@ # license: GPLv2 # -import configparser +# import configparser import glob import os import re @@ -43,7 +43,7 @@ from tools.pr_comments import get_submitted_job_comment, update_comment, make_html_list_items from tools.job_metadata import read_metadata_file -from pyghee.utils import log, error +from pyghee.utils import log AWAITS_LAUCH = "awaits_lauch" FAILURE = "failure" @@ -445,9 +445,6 @@ def process_finished_job(self, finished_job): # resources_requested=CPU:x,RAM:yG,DISK:zG # resources_allocated=CPU:x,RAM:yG,DISK:zG # resources_used=CPU:x,RAM:yG,DISK:zG - # OLD - # if file doesn't exist, use the below procedure (only until target - # repositories are updated to create such a result file) # NEW check if _bot_jobJOBID.result exits job_result_file = f"_bot_job{job_id}.result" @@ -507,12 +504,10 @@ def process_finished_job(self, finished_job): update_comment(pr_comment_id, pull_request, comment_update) - return + return - # we should only gotten here if there was no job result file or it - # could not be read. if the old code has been moved to the target - # repository we need to add a standard message ala "UNKNOWN result - # because no job result file found" + # we should not gotten here because scripts/bot-build.slurm creates + # a default results file if bot/check-result.sh doesn't exist # NOTE if also the deploy functionality is changed such to use the # results file the bot really becomes independent of what it diff --git a/tasks/build.py b/tasks/build.py index 6c4e5301..09064553 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -24,6 +24,7 @@ from retry.api import retry_call from tools import config, run_cmd from tools.job_metadata import create_metadata_file +from tools.pr_comments import PRComment APP_NAME = "app_name" AWAITS_RELEASE = "awaits_release" diff --git a/tools/job_metadata.py b/tools/job_metadata.py index da12fee4..b73448d9 100644 --- a/tools/job_metadata.py +++ b/tools/job_metadata.py @@ -13,9 +13,9 @@ import os import sys -from pyghee.utils import log, error -from tasks.build import Job -from tools.pr_comments import PRComment +from pyghee.utils import log +# from tasks.build import Job +# from tools.pr_comments import PRComment def create_metadata_file(job, job_id, pr_comment): From 20739657b9ad7efeef0e0dc52263d722fa637ce5 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 10 Apr 2023 15:01:38 +0000 Subject: [PATCH 19/41] tweaking messages generated by the bot's job script --- scripts/bot-build.slurm | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/scripts/bot-build.slurm b/scripts/bot-build.slurm index 13263b7c..498f2294 100755 --- a/scripts/bot-build.slurm +++ b/scripts/bot-build.slurm @@ -23,23 +23,27 @@ # for example, repos.cfg and configuration file bundles for repositories echo "Starting bot-build.slurm" -if [ -f bot/build.sh ]; then - echo "bot/build.sh script found in '${PWD}', so running it!" - bot/build.sh +BOT_BUILD_SCRIPT=bot/build.sh +if [ -f ${BOT_BUILD_SCRIPT} ]; then + echo "${BOT_BUILD_SCRIPT} script found in '${PWD}', so running it!" + ${BOT_BUILD_SCRIPT} else - fatal_error "could not find bot/build.sh script in '${PWD}'" + fatal_error "could not find ${BOT_BUILD_SCRIPT} script in '${PWD}'" fi echo "bot/build.sh finished" -if [ -f bot/check-result.sh ]; then - echo "bot/check-result.sh script found in '${PWD}', so running it!" - bot/check-result.sh +CHECK_RESULT_SCRIPT=_bot/check-result.sh +if [ -f ${CHECK_RESULT_SCRIPT} ]; then + echo "${CHECK_RESULT_SCRIPT} script found in '${PWD}', so running it!" + ${CHECK_RESULT_SCRIPT} else - echo "could not find bot/check-result.sh script in '${PWD}' ..." + echo "could not find ${CHECK_RESULT_SCRIPT} script in '${PWD}' ..." echo "... depositing default _bot_job${SLURM_JOB_ID}.result file in '${PWD}'" - cat > _bot_job${SLURM_JOB_ID}.result < _bot_job${SLURM_JOB_ID}.result [RESULT] -summary=UNKNOWN -details="Did not find bot/check-result.sh script in '${PWD}'. Check job manually." +summary = :shrug: UNKNOWN +details = + Did not find `bot/check-result.sh` script in job's work directory. + *Check job manually or ask an admin of the bot instance to assist you.* EOF fi echo "check result step finished" From 6127332d5180f31f45380298feb4238dc624a0e1 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 10 Apr 2023 15:03:26 +0000 Subject: [PATCH 20/41] do not include empty lines in job status lists --- tools/pr_comments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/pr_comments.py b/tools/pr_comments.py index adcd1b25..3cceb0ef 100644 --- a/tools/pr_comments.py +++ b/tools/pr_comments.py @@ -116,5 +116,6 @@ def make_html_list_items(lines): """ html_list_items = "" for line in lines.split("\n"): - html_list_items += f"
  • {line}
    • " + if len(line.strip()) > 0: + html_list_items += f"
  • {line}
    • " return html_list_items From a29d95092182cdbcd6bcd347713693847fe5633f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 10 Apr 2023 15:04:37 +0000 Subject: [PATCH 21/41] fixing bugs + tweaking some messages --- eessi_bot_job_manager.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index d5d3283b..dc70c69b 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -68,7 +68,8 @@ NEW_JOB_COMMENTS: [AWAITS_LAUCH], RUNNING_JOB_COMMENTS: [RUNNING_JOB], FINISHED_JOB_COMMENTS: [SUCCESS, FAILURE, NO_SLURM_OUT, SLURM_OUT, MISSING_MODULES, - NO_TARBALL_MESSAGE, NO_MATCHING_TARBALL, MULTIPLE_TARBALLS] + NO_TARBALL_MESSAGE, NO_MATCHING_TARBALL, MULTIPLE_TARBALLS, + JOB_RESULT_COMMENT_FMT] } @@ -455,10 +456,10 @@ def process_finished_job(self, finished_job): # get summary summary = job_results.get(JOB_RESULT_SUMMARY, ":shrug: UNKOWN") # get details - details = job_results.get(JOB_RESULT_DETAILS, "_no details provided_") + details = job_results.get(JOB_RESULT_DETAILS, "No details were provided.") details_list = make_html_list_items(details) # get built_artefacts - artefacts = job_results.get(JOB_RESULT_ARTEFACTS, "_no artefacts reported_") + artefacts = job_results.get(JOB_RESULT_ARTEFACTS, "No artefacts were found/reported.") artefacts_list = make_html_list_items(artefacts) # TODO report to log @@ -471,7 +472,8 @@ def process_finished_job(self, finished_job): dt = datetime.now(timezone.utc) - job_result_comment_fmt = config.read_config()[JOB_RESULT_COMMENT_FMT] + finished_job_comments_cfg = config.read_config()[FINISHED_JOB_COMMENTS] + job_result_comment_fmt = finished_job_comments_cfg[JOB_RESULT_COMMENT_FMT] comment_update = f"\n|{dt.strftime('%b %d %X %Z %Y')}|finished|" comment_update += job_result_comment_fmt.format( summary=summary, @@ -502,7 +504,7 @@ def process_finished_job(self, finished_job): repo = gh.get_repo(repo_name) pull_request = repo.get_pull(int(pr_number)) - update_comment(pr_comment_id, pull_request, comment_update) + update_comment(int(pr_comment_id), pull_request, comment_update) return From 517322e05db830eabc26abf561cfd3d60a1edcaf Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 10 Apr 2023 22:10:18 +0200 Subject: [PATCH 22/41] improve job result comment template --- app.cfg.example | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app.cfg.example b/app.cfg.example index d1e3c109..fe8fb0e7 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -187,7 +187,4 @@ missing_modules = Slurm output lacks message "No missing modules!". no_tarball_message = Slurm output lacks message about created tarball. no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. -job_result_comment_fmt = -
    {summary} _click triangle for details_ -Details:
      {details}
    -Artefacts:
      {artefacts}
    +job_result_comment_fmt =
    {summary} _(click triangle for detailed information)_Details:
      {details}
    Artefacts:
      {artefacts}
    From 8beab19da43f6436719d6b6fd82d2d0e2bfbe223 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 10 Apr 2023 23:22:16 +0200 Subject: [PATCH 23/41] removed leftover from testing --- scripts/bot-build.slurm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bot-build.slurm b/scripts/bot-build.slurm index 498f2294..6d8ea50a 100755 --- a/scripts/bot-build.slurm +++ b/scripts/bot-build.slurm @@ -31,7 +31,7 @@ else fatal_error "could not find ${BOT_BUILD_SCRIPT} script in '${PWD}'" fi echo "bot/build.sh finished" -CHECK_RESULT_SCRIPT=_bot/check-result.sh +CHECK_RESULT_SCRIPT=bot/check-result.sh if [ -f ${CHECK_RESULT_SCRIPT} ]; then echo "${CHECK_RESULT_SCRIPT} script found in '${PWD}', so running it!" ${CHECK_RESULT_SCRIPT} From b5b765ecfa8e171cfc0d0b96efa5f0742d80c4ce Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 14 Apr 2023 11:41:42 +0200 Subject: [PATCH 24/41] need to remove non bot jobs earlier + using current jobs for processing running jobs --- eessi_bot_job_manager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index dc70c69b..5de32b9a 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -770,6 +770,10 @@ def main(): # " %s due to parameter '--jobs %s'" % ( # nj,opts.jobs), job_manager.logfile) + # remove non bot jobs from current_jobs + for job in non_bot_jobs: + current_jobs.pop(job) + running_jobs = job_manager.determine_running_jobs(current_jobs) log( "job manager main loop: running_jobs='%s'" % @@ -779,7 +783,7 @@ def main(): for rj in running_jobs: if not job_manager.job_filter or rj in job_manager.job_filter: - job_manager.process_running_jobs(known_jobs[rj]) + job_manager.process_running_jobs(current_jobs[rj]) finished_jobs = job_manager.determine_finished_jobs( known_jobs, current_jobs) @@ -797,10 +801,6 @@ def main(): # "job %s due"" to parameter '--jobs %s'" % (fj,opts.jobs), # " job_manager.logfile)" - # remove non bot jobs from current_jobs - for job in non_bot_jobs: - current_jobs.pop(job) - known_jobs = current_jobs # sleep poll_interval seconds (only if at least one more iteration) From ef947e4e8fac9328b8318d9164cef7608b000a9b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 12:50:31 +0000 Subject: [PATCH 25/41] improvements for using results produced by bot/check-result.sh script app.cfg.example - polished/clarified notes about comment templates - more control over format for details eessi_bot_job_manager.py - support for more control over format for details in PR comment - handling case that no result file is found tools/pr_comments.py - support for more control over format for details in PR comment --- app.cfg.example | 13 ++++- eessi_bot_job_manager.py | 107 ++++++++++++++++++++++----------------- tools/pr_comments.py | 5 +- 3 files changed, 75 insertions(+), 50 deletions(-) diff --git a/app.cfg.example b/app.cfg.example index fe8fb0e7..8d9ee814 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -164,7 +164,14 @@ poll_interval = 60 scontrol_command = /usr/bin/scontrol -# variable 'comment' under 'submitted_job_comments' should not be changed as there are regular expression patterns matching it +# Note 1. The value of the setting 'initial_comment' in section +# '[submitted_job_comments]' should not be changed because the bot +# uses regular expression pattern to identify a comment with this +# format. +# Note 2. Any name inside curly brackets is replaced by the bot with +# corresponding data. If the name is changed or the curly brackets +# are removed, the output (in PR comments) will lack important +# information. [submitted_job_comments] initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}` awaits_release = job id `{job_id}` awaits release by job manager @@ -187,4 +194,6 @@ missing_modules = Slurm output lacks message "No missing modules!". no_tarball_message = Slurm output lacks message about created tarball. no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. -job_result_comment_fmt =
    {summary} _(click triangle for detailed information)_Details:
      {details}
    Artefacts:
      {artefacts}
    +job_result_comment_fmt =
    {summary} _(click triangle for detailed information)_Details:{details}Artefacts:
      {artefacts}
    +job_result_details_item_fmt =
    {item} +job_result_artefacts_item_fmt =
  • {item}
    • diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 5de32b9a..ccfdbc4d 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -60,6 +60,8 @@ SUCCESS = "success" JOB_RESULT_COMMENT_FMT = "job_result_comment_fmt" +JOB_RESULT_DETAILS_ITEM_FMT = "job_result_details_item_fmt" +JOB_RESULT_ARTEFACTS_ITEM_FMT = "job_result_artefacts_item_fmt" JOB_RESULT_SUMMARY = "summary" JOB_RESULT_DETAILS = "details" JOB_RESULT_ARTEFACTS = "artefacts" @@ -69,7 +71,8 @@ RUNNING_JOB_COMMENTS: [RUNNING_JOB], FINISHED_JOB_COMMENTS: [SUCCESS, FAILURE, NO_SLURM_OUT, SLURM_OUT, MISSING_MODULES, NO_TARBALL_MESSAGE, NO_MATCHING_TARBALL, MULTIPLE_TARBALLS, - JOB_RESULT_COMMENT_FMT] + JOB_RESULT_COMMENT_FMT, JOB_RESULT_DETAILS_ITEM_FMT, + JOB_RESULT_ARTEFACTS_ITEM_FMT] } @@ -433,7 +436,6 @@ def process_finished_job(self, finished_job): os.rename(old_symlink, new_symlink) # REPORT status (to logfile in any case, to PR comment if accessible) - # NEW # check if file _bot_jobJOBID.result exists --> if so read it and # prepare update to PR comment # result file contents: @@ -447,64 +449,77 @@ def process_finished_job(self, finished_job): # resources_allocated=CPU:x,RAM:yG,DISK:zG # resources_used=CPU:x,RAM:yG,DISK:zG - # NEW check if _bot_jobJOBID.result exits + # obtain format templates from app.cfg + finished_job_comments_cfg = config.read_config()[FINISHED_JOB_COMMENTS] + job_result_details_item_fmt = finished_job_comments_cfg[JOB_RESULT_DETAILS_ITEM_FMT] + job_result_artefacts_item_fmt = finished_job_comments_cfg[JOB_RESULT_ARTEFACTS_ITEM_FMT] + + # check if _bot_jobJOBID.result exits job_result_file = f"_bot_job{job_id}.result" job_result_file_path = os.path.join(new_symlink, job_result_file) job_results = self.read_job_result(job_result_file_path) - if job_results: - # TODO process results & return - # get summary + if job_results is None: + # set summary to ':shrug UKNOWN' + summary = ":shrug: UNKNOWN" + # set details to 'job results file ... does not exist or reading it failed' + details = f"Job results file `{job_result_file_path}` does not exist or reading it failed." + details_list = make_html_list_items(details, job_result_details_item_fmt) + artefacts = "No artefacts were found/reported." + artefacts_list = make_html_list_items(artefacts, job_result_artefacts_item_fmt) + else: + # job_results is not None + + # get summary (or set it to ':shrug: UNKNOWN' if no summary found) summary = job_results.get(JOB_RESULT_SUMMARY, ":shrug: UNKOWN") - # get details + # get details (or set it to 'No details were provided.' if no details found) details = job_results.get(JOB_RESULT_DETAILS, "No details were provided.") - details_list = make_html_list_items(details) - # get built_artefacts + details_list = make_html_list_items(details, job_result_details_item_fmt) + # get artefacts (or set it to 'No artefacts were found/reported.' if no artefacts found) artefacts = job_results.get(JOB_RESULT_ARTEFACTS, "No artefacts were found/reported.") - artefacts_list = make_html_list_items(artefacts) + artefacts_list = make_html_list_items(artefacts, job_result_artefacts_item_fmt) - # TODO report to log - log(f"{fn}(): finished job {job_id}\n" - f"########\n" - f"summary: {summary}\n" - f"details: {details}\n" - f"artefacts: {artefacts}\n" - f"########\n", self.logfile) + # TODO report to log + log(f"{fn}(): finished job {job_id}\n" + f"########\n" + f"summary: {summary}\n" + f"details: {details}\n" + f"artefacts: {artefacts}\n" + f"########\n", self.logfile) - dt = datetime.now(timezone.utc) + dt = datetime.now(timezone.utc) - finished_job_comments_cfg = config.read_config()[FINISHED_JOB_COMMENTS] - job_result_comment_fmt = finished_job_comments_cfg[JOB_RESULT_COMMENT_FMT] - comment_update = f"\n|{dt.strftime('%b %d %X %Z %Y')}|finished|" - comment_update += job_result_comment_fmt.format( - summary=summary, - details=details_list, - artefacts=artefacts_list - ) + job_result_comment_fmt = finished_job_comments_cfg[JOB_RESULT_COMMENT_FMT] + comment_update = f"\n|{dt.strftime('%b %d %X %Z %Y')}|finished|" + comment_update += job_result_comment_fmt.format( + summary=summary, + details=details_list, + artefacts=artefacts_list + ) - # obtain id of PR comment to be updated (from _bot_jobID.metadata) - metadata_file = f"_bot_job{job_id}.metadata" - job_metadata_path = os.path.join(new_symlink, metadata_file) - metadata_pr = self.read_job_pr_metadata(job_metadata_path) - if metadata_pr is None: - # TODO should we raise the Exception here? maybe first process - # the finished job and raise an exception at the end? - raise Exception("Unable to find metadata file ... skip updating PR comment") + # obtain id of PR comment to be updated (from _bot_jobID.metadata) + metadata_file = f"_bot_job{job_id}.metadata" + job_metadata_path = os.path.join(new_symlink, metadata_file) + metadata_pr = self.read_job_pr_metadata(job_metadata_path) + if metadata_pr is None: + # TODO should we raise the Exception here? maybe first process + # the finished job and raise an exception at the end? + raise Exception("Unable to find metadata file ... skip updating PR comment") - # get repo name - repo_name = metadata_pr.get("repo", None) - # get pr number - pr_number = metadata_pr.get("pr_number", -1) - # get pr comment id - pr_comment_id = metadata_pr.get("pr_comment_id", -1) - log(f"{fn}(): pr comment id {pr_comment_id}", self.logfile) + # get repo name + repo_name = metadata_pr.get("repo", None) + # get pr number + pr_number = metadata_pr.get("pr_number", -1) + # get pr comment id + pr_comment_id = metadata_pr.get("pr_comment_id", -1) + log(f"{fn}(): pr comment id {pr_comment_id}", self.logfile) - # establish contact to pull request on github - gh = github.get_instance() + # establish contact to pull request on github + gh = github.get_instance() - repo = gh.get_repo(repo_name) - pull_request = repo.get_pull(int(pr_number)) + repo = gh.get_repo(repo_name) + pull_request = repo.get_pull(int(pr_number)) - update_comment(int(pr_comment_id), pull_request, comment_update) + update_comment(int(pr_comment_id), pull_request, comment_update) return diff --git a/tools/pr_comments.py b/tools/pr_comments.py index 3cceb0ef..0362fa80 100644 --- a/tools/pr_comments.py +++ b/tools/pr_comments.py @@ -105,11 +105,12 @@ def update_pr_comment(event_info, update): issue_comment.edit(comment_new + update) -def make_html_list_items(lines): +def make_html_list_items(lines, line_format): """Makes HTML list from lines. Args: lines (string): multiline string + line_format (string): template which contains placeholder {item} Returns: multiline (string): formatted as HTML list items @@ -117,5 +118,5 @@ def make_html_list_items(lines): html_list_items = "" for line in lines.split("\n"): if len(line.strip()) > 0: - html_list_items += f"
  • {line}
    • " + html_list_items += line_format.format(item=line) return html_list_items From 05afc40c153350cd0136e7702c83768e7da8c602 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 19 Apr 2023 13:04:20 +0000 Subject: [PATCH 26/41] updating format, adding missing settings --- app.cfg.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.cfg.example b/app.cfg.example index 8d9ee814..f383bfb3 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -194,6 +194,6 @@ missing_modules = Slurm output lacks message "No missing modules!". no_tarball_message = Slurm output lacks message about created tarball. no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. -job_result_comment_fmt =
    {summary} _(click triangle for detailed information)_Details:{details}Artefacts:
      {artefacts}
    -job_result_details_item_fmt =
    {item} +job_result_comment_fmt =
    {summary} _(click triangle for detailed information)_Details:{details}Artefacts:{artefacts} +job_result_details_item_fmt =
        {item} job_result_artefacts_item_fmt =
  • {item}
    • From 52f145530d04456cc4412229df79047297bcac4a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 20 Apr 2023 09:18:59 +0200 Subject: [PATCH 27/41] polishing of job result details --- app.cfg.example | 2 +- eessi_bot_job_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app.cfg.example b/app.cfg.example index f383bfb3..b740fea6 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -194,6 +194,6 @@ missing_modules = Slurm output lacks message "No missing modules!". no_tarball_message = Slurm output lacks message about created tarball. no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. -job_result_comment_fmt =
    {summary} _(click triangle for detailed information)_Details:{details}Artefacts:{artefacts} +job_result_comment_fmt =
    {summary} _(click triangle for detailed information)_Details:{details}
    Artefacts:{artefacts} job_result_details_item_fmt =
        {item} job_result_artefacts_item_fmt =
  • {item}
    • diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index ccfdbc4d..23777e09 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -462,7 +462,7 @@ def process_finished_job(self, finished_job): # set summary to ':shrug UKNOWN' summary = ":shrug: UNKNOWN" # set details to 'job results file ... does not exist or reading it failed' - details = f"Job results file `{job_result_file_path}` does not exist or reading it failed." + details = f"Job results file `{job_result_file}` does not exist in job directory or reading it failed." details_list = make_html_list_items(details, job_result_details_item_fmt) artefacts = "No artefacts were found/reported." artefacts_list = make_html_list_items(artefacts, job_result_artefacts_item_fmt) From e9f9ae02d5b73f2e224148ed64ede4da684263e8 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 27 Apr 2023 14:00:00 +0200 Subject: [PATCH 28/41] let bot/check-result.sh provide fully defined comment details - we assume that the check-result.sh script provides a fully defined HTML string the bot can just dump as is into the column in the status table (no further processing is needed) - changes separated for each file - app.cfg.example: - removed *_fmt settings for summary, details and artefacts - added new format for when result is UNKNOWN (ie no result file found or no pre-formatted comment_details found) - eessi_bot_job_manager.py: - replaced existing constants with new constants - updated comment explaining expected format of result file - updated processing of result file (got much simpler --> complexity/work moved to target respository) - only needs to retain some code then result is not defined/unknown (format of this is defined via `app.cfg`) - scripts/bot-build.slurm: - adjusted format of result file that is generated if the target repository does not provide the bot/check-result.sh - tools/pr_comments.py: - removed function make_html_list_items which is not used/needed any longer --- app.cfg.example | 4 +-- eessi_bot_job_manager.py | 76 +++++++++++++--------------------------- scripts/bot-build.slurm | 5 +-- tools/pr_comments.py | 17 --------- 4 files changed, 26 insertions(+), 76 deletions(-) diff --git a/app.cfg.example b/app.cfg.example index b740fea6..92474337 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -194,6 +194,4 @@ missing_modules = Slurm output lacks message "No missing modules!". no_tarball_message = Slurm output lacks message about created tarball. no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. -job_result_comment_fmt =
    {summary} _(click triangle for detailed information)_Details:{details}
    Artefacts:{artefacts} -job_result_details_item_fmt =
        {item} -job_result_artefacts_item_fmt =
  • {item}
    • +job_result_unknown_fmt =
    :shrug: UNKNOWN _(click triangle for detailed information)_
    • Job results file `{filename}` does not exist in job directory or reading it failed.
    • No artefacts were found/reported.
    diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 23777e09..3d1330f2 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -40,7 +40,7 @@ from tools.args import job_manager_parse from datetime import datetime, timezone from tools import config, run_cmd -from tools.pr_comments import get_submitted_job_comment, update_comment, make_html_list_items +from tools.pr_comments import get_submitted_job_comment, update_comment from tools.job_metadata import read_metadata_file from pyghee.utils import log @@ -59,20 +59,15 @@ SLURM_OUT = "slurm_out" SUCCESS = "success" -JOB_RESULT_COMMENT_FMT = "job_result_comment_fmt" -JOB_RESULT_DETAILS_ITEM_FMT = "job_result_details_item_fmt" -JOB_RESULT_ARTEFACTS_ITEM_FMT = "job_result_artefacts_item_fmt" -JOB_RESULT_SUMMARY = "summary" -JOB_RESULT_DETAILS = "details" -JOB_RESULT_ARTEFACTS = "artefacts" +JOB_RESULT_UNKNOWN_FMT = "job_result_unknown_fmt" +JOB_RESULT_COMMENT_DETAILS = "comment_details" REQUIRED_CONFIG = { NEW_JOB_COMMENTS: [AWAITS_LAUCH], RUNNING_JOB_COMMENTS: [RUNNING_JOB], FINISHED_JOB_COMMENTS: [SUCCESS, FAILURE, NO_SLURM_OUT, SLURM_OUT, MISSING_MODULES, NO_TARBALL_MESSAGE, NO_MATCHING_TARBALL, MULTIPLE_TARBALLS, - JOB_RESULT_COMMENT_FMT, JOB_RESULT_DETAILS_ITEM_FMT, - JOB_RESULT_ARTEFACTS_ITEM_FMT] + JOB_RESULT_UNKNOWN_FMT] } @@ -436,65 +431,42 @@ def process_finished_job(self, finished_job): os.rename(old_symlink, new_symlink) # REPORT status (to logfile in any case, to PR comment if accessible) + # rely fully on what bot/check-result.sh has returned # check if file _bot_jobJOBID.result exists --> if so read it and - # prepare update to PR comment - # result file contents: - # [RESULT] - # summary={SUCCESS|FAILURE|UNKNOWN} - # details=multiline string that is put into details element - # built_artefacts=multiline string with (relative) path names - # jobid= - # runtime= - # resources_requested=CPU:x,RAM:yG,DISK:zG - # resources_allocated=CPU:x,RAM:yG,DISK:zG - # resources_used=CPU:x,RAM:yG,DISK:zG + # update PR comment + # contents of *.result file (here we only use section [RESULT]) + # [RESULT] + # comment_details = _FULLY_DEFINED_UPDATE_TO_PR_COMMENT_ + # [repo_id] + # artefacts = _LIST_OF_ARTEFACTS_TO_BE_DEPLOYED_TO_repo_id_ # obtain format templates from app.cfg finished_job_comments_cfg = config.read_config()[FINISHED_JOB_COMMENTS] - job_result_details_item_fmt = finished_job_comments_cfg[JOB_RESULT_DETAILS_ITEM_FMT] - job_result_artefacts_item_fmt = finished_job_comments_cfg[JOB_RESULT_ARTEFACTS_ITEM_FMT] # check if _bot_jobJOBID.result exits job_result_file = f"_bot_job{job_id}.result" job_result_file_path = os.path.join(new_symlink, job_result_file) job_results = self.read_job_result(job_result_file_path) - if job_results is None: - # set summary to ':shrug UKNOWN' - summary = ":shrug: UNKNOWN" - # set details to 'job results file ... does not exist or reading it failed' - details = f"Job results file `{job_result_file}` does not exist in job directory or reading it failed." - details_list = make_html_list_items(details, job_result_details_item_fmt) - artefacts = "No artefacts were found/reported." - artefacts_list = make_html_list_items(artefacts, job_result_artefacts_item_fmt) - else: - # job_results is not None - - # get summary (or set it to ':shrug: UNKNOWN' if no summary found) - summary = job_results.get(JOB_RESULT_SUMMARY, ":shrug: UNKOWN") - # get details (or set it to 'No details were provided.' if no details found) - details = job_results.get(JOB_RESULT_DETAILS, "No details were provided.") - details_list = make_html_list_items(details, job_result_details_item_fmt) - # get artefacts (or set it to 'No artefacts were found/reported.' if no artefacts found) - artefacts = job_results.get(JOB_RESULT_ARTEFACTS, "No artefacts were found/reported.") - artefacts_list = make_html_list_items(artefacts, job_result_artefacts_item_fmt) - - # TODO report to log + + # set comment_details in case no results were found (self.read_job_result + # returned None), it's also used (reused actually) in case the job + # results do not have a preformatted comment + job_result_unknown_fmt = finished_job_comments_cfg[JOB_RESULT_UNKNOWN_FMT] + comment_details = job_result_unknown_fmt.format(file=job_result_file) + if job_results: + # get preformatted comment_details or use previously set default for unknown + comment_details = job_results.get(JOB_RESULT_COMMENT_DETAILS, comment_details) + + # report to log log(f"{fn}(): finished job {job_id}\n" f"########\n" - f"summary: {summary}\n" - f"details: {details}\n" - f"artefacts: {artefacts}\n" + f"comment_details: {comment_details}\n" f"########\n", self.logfile) dt = datetime.now(timezone.utc) - job_result_comment_fmt = finished_job_comments_cfg[JOB_RESULT_COMMENT_FMT] comment_update = f"\n|{dt.strftime('%b %d %X %Z %Y')}|finished|" - comment_update += job_result_comment_fmt.format( - summary=summary, - details=details_list, - artefacts=artefacts_list - ) + comment_update += f"{comment_details}|" # obtain id of PR comment to be updated (from _bot_jobID.metadata) metadata_file = f"_bot_job{job_id}.metadata" diff --git a/scripts/bot-build.slurm b/scripts/bot-build.slurm index 6d8ea50a..b80866af 100755 --- a/scripts/bot-build.slurm +++ b/scripts/bot-build.slurm @@ -40,10 +40,7 @@ else echo "... depositing default _bot_job${SLURM_JOB_ID}.result file in '${PWD}'" cat << 'EOF' > _bot_job${SLURM_JOB_ID}.result [RESULT] -summary = :shrug: UNKNOWN -details = - Did not find `bot/check-result.sh` script in job's work directory. - *Check job manually or ask an admin of the bot instance to assist you.* +comment_details =
    :shrug: UNKNOWN _(click triangle for detailed information)_
    • Did not find `bot/check-result.sh` script in job's work directory.
    • *Check job manually or ask an admin of the bot instance to assist you.*
    EOF fi echo "check result step finished" diff --git a/tools/pr_comments.py b/tools/pr_comments.py index 0362fa80..956e990c 100644 --- a/tools/pr_comments.py +++ b/tools/pr_comments.py @@ -103,20 +103,3 @@ def update_pr_comment(event_info, update): pull_request = repo.get_pull(pr_number) issue_comment = pull_request.get_issue_comment(issue_id) issue_comment.edit(comment_new + update) - - -def make_html_list_items(lines, line_format): - """Makes HTML list from lines. - - Args: - lines (string): multiline string - line_format (string): template which contains placeholder {item} - - Returns: - multiline (string): formatted as HTML list items - """ - html_list_items = "" - for line in lines.split("\n"): - if len(line.strip()) > 0: - html_list_items += line_format.format(item=line) - return html_list_items From ee3a63185b0c01b969b43d0798770e118695deda Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 27 Apr 2023 14:45:28 +0200 Subject: [PATCH 29/41] removed all software-layer specific code that checked result of a build job --- eessi_bot_job_manager.py | 163 --------------------------------------- 1 file changed, 163 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 3d1330f2..2dc07414 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -29,7 +29,6 @@ # # import configparser -import glob import os import re import time @@ -495,168 +494,6 @@ def process_finished_job(self, finished_job): return - # we should not gotten here because scripts/bot-build.slurm creates - # a default results file if bot/check-result.sh doesn't exist - - # NOTE if also the deploy functionality is changed such to use the - # results file the bot really becomes independent of what it - # builds - - # TODO the below should be done by the target repository's script - # bot/check-result.sh which should produce a file - # _bot_jobJOBID.result - # check result - # ("No missing packages!", "eessi-.*.tar.gz") - # TODO as is, this requires knowledge about the build process. - # maybe better to somehow capture job "result" (process - # exit value) by build script? - # update PR comment with new status (finished) - # move symlink from job_ids_dir/submitted to jobs_ids_dir/finished - - # 'submitted_jobs_dir'/jobid is symlink to working dir of job - # working dir contains _bot_job.metadata - # file contains (pr number and base repo name) - - # establish contact to pull request on github - gh = github.get_instance() - - # set some variables for accessing work dir of job - sym_dst = os.readlink(new_symlink) - - # read some information from job metadata file - metadata_file = f"_bot_job{job_id}.metadata" - job_metadata_path = os.path.join(new_symlink, metadata_file) - - # check if metadata file exist - metadata_pr = self.read_job_pr_metadata(job_metadata_path) - if metadata_pr is None: - # TODO should we raise the Exception here? maybe first process - # the finished job and raise an exception at the end? - raise Exception("Unable to find metadata file") - - # get repo name - repo_name = metadata_pr.get("repo", "") - # get pr number - pr_number = metadata_pr.get("pr_number", None) - - repo = gh.get_repo(repo_name) - pull_request = repo.get_pull(int(pr_number)) - - # determine comment to be updated - if "comment_id" not in finished_job: - finished_job_cmnt = get_submitted_job_comment(pull_request, job_id) - - if finished_job_cmnt: - log(f"{fn}(): found comment with id {finished_job_cmnt.id}", self.logfile) - finished_job["comment_id"] = finished_job_cmnt.id - - # analyse job result - slurm_out = os.path.join(sym_dst, "slurm-{job_id}.out") - - # determine all tarballs that are stored in - # the job directory (only expecting 1) - tarball_pattern = "eessi-*software-*.tar.gz" - glob_str = os.path.join(sym_dst, tarball_pattern) - eessi_tarballs = glob.glob(glob_str) - - # set some initial values - no_missing_modules = False - targz_created = False - - # check slurm out for the below strings - # ^No missing modules!$ --> software successfully installed - # ^/eessi_bot_job/eessi-.*-software-.*.tar.gz - # created!$ --> tarball successfully created - if os.path.exists(slurm_out): - re_missing_modules = re.compile("^No missing modules!$") - re_targz_created = re.compile( - "^/eessi_bot_job/eessi-.*-software-.*.tar.gz created!$" - ) - outfile = open(slurm_out, "r") - for line in outfile: - if re_missing_modules.match(line): - # no missing modules - no_missing_modules = True - if re_targz_created.match(line): - # tarball created - targz_created = True - - dt = datetime.now(timezone.utc) - - finished_job_comments_cfg = config.read_config()[FINISHED_JOB_COMMENTS] - comment_update = f"\n|{dt.strftime('%b %d %X %Z %Y')}|finished|" - if (no_missing_modules and targz_created and - len(eessi_tarballs) == 1): - # We've got one tarball and slurm out messages are ok - # Prepare a message with information such as - # (installation status, tarball name, tarball size) - tarball_name = os.path.basename(eessi_tarballs[0]) - tarball_size = os.path.getsize(eessi_tarballs[0]) / 2**30 - success_comment = finished_job_comments_cfg[SUCCESS].format( - tarball_name=tarball_name, - tarball_size=tarball_size - ) - comment_update += f"{success_comment}|" - # NOTE explicitly name repo in build job comment? - # comment_update += '\nAwaiting approval to - # comment_update += ingest tarball into the repository.' - else: - # something is not allright: - # - no slurm out or - # - did not find the messages we expect or - # - no tarball or - # - more than one tarball - # prepare a message with details about the above conditions and - # update PR with a comment - - comment_update += f"{finished_job_comments_cfg[FAILURE]}
      " - found_slurm_out = os.path.exists(slurm_out) - - if not found_slurm_out: - # no slurm out ... something went wrong with the job f"
    • {}
      • " - comment_update += f"
    • {finished_job_comments_cfg[NO_SLURM_OUT]}
      • ".format( - slurm_out=os.path.basename(slurm_out) - ) - else: - comment_update += f"
    • {finished_job_comments_cfg[SLURM_OUT]}
      • ".format( - slurm_out=os.path.basename(slurm_out) - ) - - if found_slurm_out and not no_missing_modules: - # Found slurm out, but doesn't contain message 'No missing modules!' - comment_update += f"
    • {finished_job_comments_cfg[MISSING_MODULES]}
      • " - - if found_slurm_out and not targz_created: - # Found slurm out, but doesn't contain message - # 'eessi-.*-software-.*.tar.gz created!' - comment_update += f"
    • {finished_job_comments_cfg[NO_TARBALL_MESSAGE]}
      • " - - if len(eessi_tarballs) == 0: - # no luck, job just seemed to have failed ... - comment_update += f"
    • {finished_job_comments_cfg[NO_MATCHING_TARBALL]}
      • ".format( - tarball_pattern=tarball_pattern.replace(r"*", r"\*") - ) - - if len(eessi_tarballs) > 1: - # something's fishy, we only expected a single tar.gz file - comment_update += f"
    • {finished_job_comments_cfg[MULTIPLE_TARBALLS]}
      • ".format( - num_tarballs=len(eessi_tarballs), - tarball_pattern=tarball_pattern.replace(r"*", r"\*") - ) - comment_update += "
    |" - # comment_update += '\nAn admin may investigate what went wrong. - # comment_update += (TODO implement procedure to ask for - # comment_update += details by adding a command to this comment.)' - - # (c) add a row to the table - # add row to status table if we found a comment - if "comment_id" in finished_job: - update_comment(finished_job["comment_id"], pull_request, comment_update) - else: - job_id = finished_job["jobid"] - log(f"{fn}(): did not find a comment for job {job_id}", self.logfile) - # TODO just create one? - def main(): """Main function.""" From ace99ec933c54376017ff7fd9757709b492ee093 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 28 Apr 2023 13:52:03 +0200 Subject: [PATCH 30/41] renamed comment_details to comment_description + clarification of result file format --- eessi_bot_job_manager.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 2dc07414..85b6c920 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -59,7 +59,7 @@ SUCCESS = "success" JOB_RESULT_UNKNOWN_FMT = "job_result_unknown_fmt" -JOB_RESULT_COMMENT_DETAILS = "comment_details" +JOB_RESULT_COMMENT_DESCRIPTION = "comment_description" REQUIRED_CONFIG = { NEW_JOB_COMMENTS: [AWAITS_LAUCH], @@ -435,9 +435,9 @@ def process_finished_job(self, finished_job): # update PR comment # contents of *.result file (here we only use section [RESULT]) # [RESULT] - # comment_details = _FULLY_DEFINED_UPDATE_TO_PR_COMMENT_ - # [repo_id] - # artefacts = _LIST_OF_ARTEFACTS_TO_BE_DEPLOYED_TO_repo_id_ + # comment_description = _FULLY_DEFINED_UPDATE_TO_PR_COMMENT_ + # status = {SUCCESS,FAILURE,UNKNOWN} + # artefacts = _LIST_OF_ARTEFACTS_TO_BE_DEPLOYED_ # obtain format templates from app.cfg finished_job_comments_cfg = config.read_config()[FINISHED_JOB_COMMENTS] @@ -447,25 +447,25 @@ def process_finished_job(self, finished_job): job_result_file_path = os.path.join(new_symlink, job_result_file) job_results = self.read_job_result(job_result_file_path) - # set comment_details in case no results were found (self.read_job_result + # set comment_description in case no results were found (self.read_job_result # returned None), it's also used (reused actually) in case the job # results do not have a preformatted comment job_result_unknown_fmt = finished_job_comments_cfg[JOB_RESULT_UNKNOWN_FMT] - comment_details = job_result_unknown_fmt.format(file=job_result_file) + comment_description = job_result_unknown_fmt.format(file=job_result_file) if job_results: - # get preformatted comment_details or use previously set default for unknown - comment_details = job_results.get(JOB_RESULT_COMMENT_DETAILS, comment_details) + # get preformatted comment_description or use previously set default for unknown + comment_description = job_results.get(JOB_RESULT_COMMENT_DESCRIPTION, comment_description) # report to log log(f"{fn}(): finished job {job_id}\n" f"########\n" - f"comment_details: {comment_details}\n" + f"comment_description: {comment_description}\n" f"########\n", self.logfile) dt = datetime.now(timezone.utc) comment_update = f"\n|{dt.strftime('%b %d %X %Z %Y')}|finished|" - comment_update += f"{comment_details}|" + comment_update += f"{comment_description}|" # obtain id of PR comment to be updated (from _bot_jobID.metadata) metadata_file = f"_bot_job{job_id}.metadata" From 03d9472ebd7cce7f417488a66c39e249cb4b2582 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 17 Jun 2023 10:18:08 +0200 Subject: [PATCH 31/41] remove unused import --- tests/test_task_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_task_build.py b/tests/test_task_build.py index 428cbfb9..261db9e3 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -27,7 +27,7 @@ # Local application imports (anything from EESSI/eessi-bot-software-layer) from tasks.build import Job, create_pr_comment from tools import run_cmd, run_subprocess -from tools.pr_comments import get_submitted_job_comment, PRComment +from tools.pr_comments import get_submitted_job_comment from tools.job_metadata import create_metadata_file # Local tests imports (reusing code from other tests) From c43c04ad206fc4be95e60513b38dc62f3dec53c7 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 03:46:41 +0000 Subject: [PATCH 32/41] fix arg name --- eessi_bot_job_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index d96059a2..b65a92b3 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -451,7 +451,7 @@ def process_finished_job(self, finished_job): # returned None), it's also used (reused actually) in case the job # results do not have a preformatted comment job_result_unknown_fmt = finished_job_comments_cfg[JOB_RESULT_UNKNOWN_FMT] - comment_description = job_result_unknown_fmt.format(file=job_result_file) + comment_description = job_result_unknown_fmt.format(filename=job_result_file) if job_results: # get preformatted comment_description or use previously set default for unknown comment_description = job_results.get(JOB_RESULT_COMMENT_DESCRIPTION, comment_description) From b67d17d33c2dbe82475a5be55e2f0fe1cdafd1f7 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 13:11:29 +0200 Subject: [PATCH 33/41] update search pattern for No missing ... --- tasks/deploy.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tasks/deploy.py b/tasks/deploy.py index 60144dc9..fb1c6d65 100644 --- a/tasks/deploy.py +++ b/tasks/deploy.py @@ -110,6 +110,12 @@ def check_build_status(slurm_out, eessi_tarballs): Returns: (bool): True -> job succeeded, False -> job failed """ + # TODO use _bot_job.result file to determine result status + # cases: + # (1) no result file --> add line with unknown status, found tarball xyz but no result file + # (2) result file && status = SUCCESS --> return True + # (3) result file && status = FAILURE --> return False + # Function checks if all modules have been built and if a tarball has # been created. @@ -122,7 +128,7 @@ def check_build_status(slurm_out, eessi_tarballs): # ^/eessi_bot_job/eessi-.*-software-.*.tar.gz created!$ --> # tarball successfully created if os.path.exists(slurm_out): - re_missing_modules = re.compile("^No missing modules!$") + re_missing_modules = re.compile("^No missing installations") re_targz_created = re.compile("^/eessi_bot_job/eessi-.*-software-.*.tar.gz created!$") outfile = open(slurm_out, "r") for line in outfile: From e67f7e8fd22739e6cc2d056d371cc9e27120d0a5 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 11:35:34 +0000 Subject: [PATCH 34/41] add more logging + fix search pattern --- tasks/deploy.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tasks/deploy.py b/tasks/deploy.py index fb1c6d65..b3b98209 100644 --- a/tasks/deploy.py +++ b/tasks/deploy.py @@ -110,6 +110,8 @@ def check_build_status(slurm_out, eessi_tarballs): Returns: (bool): True -> job succeeded, False -> job failed """ + fn = sys._getframe().f_code.co_name + # TODO use _bot_job.result file to determine result status # cases: # (1) no result file --> add line with unknown status, found tarball xyz but no result file @@ -128,16 +130,20 @@ def check_build_status(slurm_out, eessi_tarballs): # ^/eessi_bot_job/eessi-.*-software-.*.tar.gz created!$ --> # tarball successfully created if os.path.exists(slurm_out): - re_missing_modules = re.compile("^No missing installations") + re_missing_modules = re.compile("No missing installations") re_targz_created = re.compile("^/eessi_bot_job/eessi-.*-software-.*.tar.gz created!$") outfile = open(slurm_out, "r") for line in outfile: if re_missing_modules.match(line): # no missing modules no_missing_modules = True + log(f"{fn}(): line '{line}' matches 'No missing installations'") if re_targz_created.match(line): # tarball created targz_created = True + log(f"{fn}(): line '{line}' matches '^/eessi_bot_job/eessi-.*-software-.*.tar.gz created!$'") + + log(f"{fn}(): found {len(eessi_tarballs)} tarballs for '{slurm_out}'") # we test results from the above check and if there is one tarball only if no_missing_modules and targz_created and len(eessi_tarballs) == 1: From c1f3d3d338b857a69bdb30f32d30e75fed12e9d3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 18 Jun 2023 16:13:33 +0000 Subject: [PATCH 35/41] fix pattern to check for missing installations --- tasks/deploy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/deploy.py b/tasks/deploy.py index b3b98209..9b80ebfe 100644 --- a/tasks/deploy.py +++ b/tasks/deploy.py @@ -130,14 +130,14 @@ def check_build_status(slurm_out, eessi_tarballs): # ^/eessi_bot_job/eessi-.*-software-.*.tar.gz created!$ --> # tarball successfully created if os.path.exists(slurm_out): - re_missing_modules = re.compile("No missing installations") + re_missing_modules = re.compile(".*No missing installations, party time!.*") re_targz_created = re.compile("^/eessi_bot_job/eessi-.*-software-.*.tar.gz created!$") outfile = open(slurm_out, "r") for line in outfile: if re_missing_modules.match(line): # no missing modules no_missing_modules = True - log(f"{fn}(): line '{line}' matches 'No missing installations'") + log(f"{fn}(): line '{line}' matches '.*No missing installations, party time!.*'") if re_targz_created.match(line): # tarball created targz_created = True From 34021f2d692450e1615f292e88c6a560743e55a0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 19 Jun 2023 18:42:05 +0200 Subject: [PATCH 36/41] trivial import fixes --- eessi_bot_job_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index b65a92b3..fb8e32be 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -28,19 +28,18 @@ # license: GPLv2 # -# import configparser import os import re -import time import sys +import time from connections import github from tools.args import job_manager_parse from datetime import datetime, timezone from tools import config, run_cmd -from tools.pr_comments import get_submitted_job_comment, update_comment from tools.job_metadata import read_metadata_file +from tools.pr_comments import get_submitted_job_comment, update_comment from pyghee.utils import log From a9f8cb21ee9c86e0eed541103f9206e6e2ee0e54 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 19 Jun 2023 19:13:52 +0200 Subject: [PATCH 37/41] fix test_create_metadata_file --- tests/test_task_build.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/test_task_build.py b/tests/test_task_build.py index 52d2e829..78d7b2e6 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -27,8 +27,8 @@ # Local application imports (anything from EESSI/eessi-bot-software-layer) from tasks.build import Job, create_pr_comment from tools import run_cmd, run_subprocess -from tools.pr_comments import get_submitted_job_comment from tools.job_metadata import create_metadata_file +from tools.pr_comments import PRComment, get_submitted_job_comment # Local tests imports (reusing code from other tests) from tests.test_tools_pr_comments import MockIssueComment @@ -412,10 +412,9 @@ def test_create_metadata_file(mocked_github, tmpdir): job_id = "123" repo_name = "test_repo" - pr_comment = MockIssueComment("test_create_metadata_file", comment_id=77) + pr_comment = PRComment(repo_name, pr_number, 77) repo = mocked_github.get_repo(repo_name) - pr = repo.get_pull(pr_number) - create_metadata_file(job, job_id, pr, pr_comment) + create_metadata_file(job, job_id, pr_comment) expected_file = f"_bot_job{job_id}.metadata" expected_file_path = os.path.join(tmpdir, expected_file) @@ -435,14 +434,14 @@ def test_create_metadata_file(mocked_github, tmpdir): job2 = Job(dir_does_not_exist, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) job_id2 = "222" with pytest.raises(FileNotFoundError): - create_metadata_file(job2, job_id2, pr, pr_comment) + create_metadata_file(job2, job_id2, pr_comment) # use directory without write permission dir_without_write_perm = os.path.join("/") job3 = Job(dir_without_write_perm, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) job_id3 = "333" with pytest.raises(OSError): - create_metadata_file(job3, job_id3, pr, pr_comment) + create_metadata_file(job3, job_id3, pr_comment) # disk quota exceeded (difficult to create and unlikely to happen because # partition where file is stored is usually very large) @@ -451,7 +450,7 @@ def test_create_metadata_file(mocked_github, tmpdir): # job_id = None job4 = Job(tmpdir, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) job_id4 = None - create_metadata_file(job4, job_id4, pr, pr_comment) + create_metadata_file(job4, job_id4, pr_comment) expected_file4 = f"_bot_job{job_id}.metadata" expected_file_path4 = os.path.join(tmpdir, expected_file4) @@ -467,4 +466,4 @@ def test_create_metadata_file(mocked_github, tmpdir): job5 = Job(None, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) job_id5 = "555" with pytest.raises(TypeError): - create_metadata_file(job5, job_id5, pr, pr_comment) + create_metadata_file(job5, job_id5, pr_comment) From 625da33fbe72bb707def522b7f385ecb675af835 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 19 Jun 2023 19:21:02 +0200 Subject: [PATCH 38/41] also test use of read_metadata_file function --- tests/test_task_build.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/test_task_build.py b/tests/test_task_build.py index 78d7b2e6..6233c4b0 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -27,7 +27,7 @@ # Local application imports (anything from EESSI/eessi-bot-software-layer) from tasks.build import Job, create_pr_comment from tools import run_cmd, run_subprocess -from tools.job_metadata import create_metadata_file +from tools.job_metadata import create_metadata_file, read_metadata_file from tools.pr_comments import PRComment, get_submitted_job_comment # Local tests imports (reusing code from other tests) @@ -402,7 +402,7 @@ def test_create_pr_comment_three_raises(mocked_github, tmpdir): @pytest.mark.repo_name("test_repo") @pytest.mark.pr_number(999) -def test_create_metadata_file(mocked_github, tmpdir): +def test_create_read_metadata_file(mocked_github, tmpdir): """Tests for function create_metadata_file.""" # create some test data ym = datetime.today().strftime('%Y.%m') @@ -429,6 +429,14 @@ def test_create_metadata_file(mocked_github, tmpdir): test_file = "tests/test_bot_job123.metadata" assert filecmp.cmp(expected_file_path, test_file, shallow=False) + # also check reading back of metadata file + metadata = read_metadata_file(expected_file_path) + assert "PR" in metadata + assert metadata["PR"]["repo"] == "test_repo" + assert metadata["PR"]["pr_number"] == "999" + assert metadata["PR"]["pr_comment_id"] == "77" + assert sorted(metadata["PR"].keys()) == ["pr_comment_id", "pr_number", "repo"] + # use directory that does not exist dir_does_not_exist = os.path.join(tmpdir, "dir_does_not_exist") job2 = Job(dir_does_not_exist, "test/architecture", "EESSI-pilot", "--speed_up_job", ym, pr_number) From 82ac4ae9a2016e10a072b5ed75f231738aa05566 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 19 Jun 2023 21:11:55 +0200 Subject: [PATCH 39/41] fix fetching PR comment ID in submit_build_jobs --- tasks/build.py | 2 +- tests/test_task_build.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tasks/build.py b/tasks/build.py index a0d1dd1b..b9d3dfda 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -653,7 +653,7 @@ def submit_build_jobs(pr, event_info, action_filter): pr_comment = create_pr_comment(job, job_id, app_name, pr, gh, symlink) job_id_to_comment_map[job_id] = pr_comment - pr_comment = pr_comments.PRComment(pr.base.repo.full_name, pr.number, pr_comment_id) + pr_comment = pr_comments.PRComment(pr.base.repo.full_name, pr.number, pr_comment.id) # create _bot_job.metadata file in submission directory create_metadata_file(job, job_id, pr_comment) diff --git a/tests/test_task_build.py b/tests/test_task_build.py index 6233c4b0..d1dd415d 100644 --- a/tests/test_task_build.py +++ b/tests/test_task_build.py @@ -413,7 +413,6 @@ def test_create_read_metadata_file(mocked_github, tmpdir): repo_name = "test_repo" pr_comment = PRComment(repo_name, pr_number, 77) - repo = mocked_github.get_repo(repo_name) create_metadata_file(job, job_id, pr_comment) expected_file = f"_bot_job{job_id}.metadata" From c627f166f35c841834eaa4fba9d180f09968f757 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 20 Jun 2023 11:21:56 +0200 Subject: [PATCH 40/41] use `comment_description` rather than `comment_description` in `*.result` file created by `bot-build.slurm` when no `bot/check-result.sh` script was found --- scripts/bot-build.slurm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bot-build.slurm b/scripts/bot-build.slurm index 188ecb34..264539ed 100755 --- a/scripts/bot-build.slurm +++ b/scripts/bot-build.slurm @@ -41,7 +41,7 @@ else echo "... depositing default _bot_job${SLURM_JOB_ID}.result file in '${PWD}'" cat << 'EOF' > _bot_job${SLURM_JOB_ID}.result [RESULT] -comment_details =
    :shrug: UNKNOWN _(click triangle for detailed information)_
    • Did not find `bot/check-result.sh` script in job's work directory.
    • *Check job manually or ask an admin of the bot instance to assist you.*
    +comment_description =
    :shrug: UNKNOWN _(click triangle for detailed information)_
    • Did not find `bot/check-result.sh` script in job's work directory.
    • *Check job manually or ask an admin of the bot instance to assist you.*
    EOF fi echo "check result step finished" From 23b1e998adaa3b7f82e5eabb2eb9f23d522b2791 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 20 Jun 2023 12:05:18 +0200 Subject: [PATCH 41/41] added missing settings to job result file created by bot-build.slurm --- scripts/bot-build.slurm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/bot-build.slurm b/scripts/bot-build.slurm index 264539ed..3f617fbb 100755 --- a/scripts/bot-build.slurm +++ b/scripts/bot-build.slurm @@ -42,6 +42,8 @@ else cat << 'EOF' > _bot_job${SLURM_JOB_ID}.result [RESULT] comment_description =
    :shrug: UNKNOWN _(click triangle for detailed information)_
    • Did not find `bot/check-result.sh` script in job's work directory.
    • *Check job manually or ask an admin of the bot instance to assist you.*
    +status = UNKNOWN +artefacts = EOF fi echo "check result step finished"