From 352918265f77a41800a429c52148c9dd0ab794b6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 13 Oct 2023 21:25:08 +0200 Subject: [PATCH 1/9] add steps for running and checking tests after a build ended --- scripts/bot-build.slurm | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/scripts/bot-build.slurm b/scripts/bot-build.slurm index 256286b0..b626c21f 100755 --- a/scripts/bot-build.slurm +++ b/scripts/bot-build.slurm @@ -47,3 +47,26 @@ artefacts = EOF fi echo "check result step finished" +TEST_SCRIPT=bot/test.sh +if [ -f ${TEST_SCRIPT} ]; then + echo "${TEST_SCRIPT} script found in '${PWD}', so running it!" + ${TEST_SCRIPT} + echo "${TEST_SCRIPT} finished" +else + echo "could not find ${TEST_SCRIPT} script in '${PWD}'" >&2 +fi +CHECK_TEST_SCRIPT=bot/check-test.sh +if [ -f ${CHECK_TEST_SCRIPT} ]; then + echo "${CHECK_TEST_SCRIPT} script found in '${PWD}', so running it!" + ${CHECK_TEST_SCRIPT} +else + echo "could not find ${CHECK_TEST_SCRIPT} script in '${PWD}' ..." + echo "... depositing default _bot_job${SLURM_JOB_ID}.test file in '${PWD}'" + cat << 'EOF' > _bot_job${SLURM_JOB_ID}.test +[RESULT] +comment_description =
:shrug: UNKNOWN _(click triangle for detailed information)_
  • Did not find `bot/check-test.sh` script in job's work directory.
  • *Check job manually or ask an admin of the bot instance to assist you.*
+status = UNKNOWN +artefacts = +EOF +fi +echo "check test step finished" From 5f2f32202a9f5b2d7bdf2c34608da3d8a9206a6a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 13 Oct 2023 21:46:42 +0200 Subject: [PATCH 2/9] add code to process results of test suite run --- eessi_bot_job_manager.py | 73 +++++++++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 9 deletions(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 30be3626..71619d5e 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -51,6 +51,8 @@ FINISHED_JOB_COMMENTS = "finished_job_comments" JOB_RESULT_COMMENT_DESCRIPTION = "comment_description" JOB_RESULT_UNKNOWN_FMT = "job_result_unknown_fmt" +JOB_TEST_COMMENT_DESCRIPTION = "comment_description" +JOB_TEST_UNKNOWN_FMT = "job_test_unknown_fmt" MISSING_MODULES = "missing_modules" MULTIPLE_TARBALLS = "multiple_tarballs" NEW_JOB_COMMENTS = "new_job_comments" @@ -285,6 +287,24 @@ def read_job_result(self, job_result_file_path): else: return None + def read_job_test(self, job_test_file_path): + """ + Read job test file and return the contents of the 'TEST' section. + + Args: + job_test_file_path (string): path to job test file + + Returns: + (ConfigParser): instance of ConfigParser corresponding to the + 'TEST' section or None + """ + # reuse function from module tools.job_metadata to read metadata file + test = read_metadata_file(job_test_file_path, self.logfile) + if test and "TEST" in test: + return result["TEST"] + else: + return None + def process_new_job(self, new_job): """ Process a new job by verifying that it is a bot job and if so @@ -470,7 +490,8 @@ def process_finished_job(self, finished_job): """ Process a finished job by - moving the symlink to the directory storing finished jobs, - - updating the PR comment with information from '*.result' file + - updating the PR comment with information from '*.result' and '*.test' + files Args: finished_job (dict): dictionary with information about the job @@ -497,14 +518,21 @@ def process_finished_job(self, finished_job): os.rename(old_symlink, new_symlink) # REPORT status (to logfile in any case, to PR comment if accessible) - # rely fully on what bot/check-build.sh has returned - # check if file _bot_jobJOBID.result exists --> if so read it and - # update PR comment - # contents of *.result file (here we only use section [RESULT]) - # [RESULT] - # comment_description = _FULLY_DEFINED_UPDATE_TO_PR_COMMENT_ - # status = {SUCCESS,FAILURE,UNKNOWN} - # artefacts = _LIST_OF_ARTEFACTS_TO_BE_DEPLOYED_ + # - rely fully on what bot/check-build.sh and bot/check-test.sh have + # returned + # - check if file _bot_jobJOBID.result exists --> if so read it and + # update PR comment + # . contents of *.result file (here we only use section [RESULT]) + # [RESULT] + # comment_description = _FULLY_DEFINED_UPDATE_TO_PR_COMMENT_ + # status = {SUCCESS,FAILURE,UNKNOWN} + # artefacts = _LIST_OF_ARTEFACTS_TO_BE_DEPLOYED_ + # - check if file _bot_jobJOBID.test exists --> if so read it and + # update PR comment + # . contents of *.test file (here we only use section [TEST]) + # [TEST] + # comment_description = _FULLY_DEFINED_UPDATE_TO_PR_COMMENT_ + # status = {SUCCESS,FAILURE,UNKNOWN} # obtain format templates from app.cfg finished_job_comments_cfg = config.read_config()[FINISHED_JOB_COMMENTS] @@ -533,6 +561,33 @@ def process_finished_job(self, finished_job): comment_update = f"\n|{dt.strftime('%b %d %X %Z %Y')}|finished|" comment_update += f"{comment_description}|" + # check if _bot_jobJOBID.test exits + # TODO if not found, assume test was not run (or failed, or ...) and add + # a message noting that ('not tested' + 'test suite not run or failed') + # --> 'bot: test [FILTERs]' command runs tests when filter matches + job_test_file = f"_bot_job{job_id}.test" + job_test_file_path = os.path.join(new_symlink, job_test_file) + job_tests = self.read_job_test(job_test_file_path) + + job_test_unknown_fmt = finished_job_comments_cfg[JOB_TEST_UNKNOWN_FMT] + # set fallback comment_description in case no test file was found + # (self.read_job_result returned None) + comment_description = job_test_unknown_fmt.format(filename=job_test_file) + if job_tests: + # get preformatted comment_description or use previously set default for unknown + comment_description = job_tests.get(JOB_TEST_COMMENT_DESCRIPTION, comment_description) + + # report to log + log(f"{fn}(): finished job {job_id}, test suite result\n" + f"########\n" + f"comment_description: {comment_description}\n" + f"########\n", self.logfile) + + dt = datetime.now(timezone.utc) + + comment_update += f"\n|{dt.strftime('%b %d %X %Z %Y')}|tested|" + comment_update += f"{comment_description}|" + # obtain id of PR comment to be updated (from file '_bot_jobID.metadata') metadata_file = f"_bot_job{job_id}.metadata" job_metadata_path = os.path.join(new_symlink, metadata_file) From 37eec6732420300b4c6fd7a6ccfae837545bf4c3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 13 Oct 2023 21:48:14 +0200 Subject: [PATCH 3/9] add message template used when no result of test suite is found --- README.md | 6 ++++++ app.cfg.example | 1 + 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index 317dc921..505ea78c 100644 --- a/README.md +++ b/README.md @@ -600,6 +600,12 @@ job_result_unknown_fmt =
:shrug: UNKNOWN _(click triangle for `job_result_unknown_fmt` is used in case no result file (produced by `bot/check-build.sh` provided by target repository) was found. +``` +job_test_unknown_fmt =
:shrug: UNKNOWN _(click triangle for details)_
  • Job test file `{filename}` does not exist in job directory or reading it failed.
+``` +`job_test_unknown_fmt` is used in case no test file (produced by `bot/check-test.sh` +provided by target repository) was found. + # Instructions to run the bot components The bot consists of three components: diff --git a/app.cfg.example b/app.cfg.example index 922df7ae..7f4f40ee 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -215,3 +215,4 @@ no_tarball_message = Slurm output lacks message about created tarball. no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. job_result_unknown_fmt =
:shrug: UNKNOWN _(click triangle for detailed information)_
  • Job results file `{filename}` does not exist in job directory or reading it failed.
  • No artefacts were found/reported.
+job_test_unknown_fmt =
:shrug: UNKNOWN _(click triangle for detailed information)_
  • Job test file `{filename}` does not exist in job directory or reading it failed.
From 6357e01ff4bbf98a75279150cfda80d91d7cd8fd Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 14 Oct 2023 20:13:29 +0200 Subject: [PATCH 4/9] fix hound issue --- eessi_bot_job_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 71619d5e..6df9de4d 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -301,7 +301,7 @@ def read_job_test(self, job_test_file_path): # reuse function from module tools.job_metadata to read metadata file test = read_metadata_file(job_test_file_path, self.logfile) if test and "TEST" in test: - return result["TEST"] + return test["TEST"] else: return None From ac62352a37db7ab147eff9a3d3eb14ce3573cad3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 26 Nov 2023 12:12:27 +0100 Subject: [PATCH 5/9] change "check result step finished" to "check build step finished" --- scripts/bot-build.slurm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bot-build.slurm b/scripts/bot-build.slurm index b626c21f..bb0faa91 100755 --- a/scripts/bot-build.slurm +++ b/scripts/bot-build.slurm @@ -46,7 +46,7 @@ status = UNKNOWN artefacts = EOF fi -echo "check result step finished" +echo "check build step finished" TEST_SCRIPT=bot/test.sh if [ -f ${TEST_SCRIPT} ]; then echo "${TEST_SCRIPT} script found in '${PWD}', so running it!" From b279e1d1e1e550cbd3e3cd1ec5dde536691e6525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Sun, 26 Nov 2023 12:27:07 +0100 Subject: [PATCH 6/9] clarify comments on tests Co-authored-by: Kenneth Hoste --- eessi_bot_job_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index 6df9de4d..afe41a5f 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -564,7 +564,7 @@ def process_finished_job(self, finished_job): # check if _bot_jobJOBID.test exits # TODO if not found, assume test was not run (or failed, or ...) and add # a message noting that ('not tested' + 'test suite not run or failed') - # --> 'bot: test [FILTERs]' command runs tests when filter matches + # --> bot/test.sh and bot/check-test.sh scripts are run in job script used by bot for 'build' action job_test_file = f"_bot_job{job_id}.test" job_test_file_path = os.path.join(new_symlink, job_test_file) job_tests = self.read_job_test(job_test_file_path) From 57bbb6919c3add1487447f550a8877195b5dd7de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Sun, 26 Nov 2023 14:40:46 +0100 Subject: [PATCH 7/9] make message for processing test files more clear Co-authored-by: Kenneth Hoste --- README.md | 2 +- app.cfg.example | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 505ea78c..1600f958 100644 --- a/README.md +++ b/README.md @@ -601,7 +601,7 @@ job_result_unknown_fmt =
:shrug: UNKNOWN _(click triangle for provided by target repository) was found. ``` -job_test_unknown_fmt =
:shrug: UNKNOWN _(click triangle for details)_
  • Job test file `{filename}` does not exist in job directory or reading it failed.
+job_test_unknown_fmt =
:shrug: UNKNOWN _(click triangle for details)_
  • Job test file `{filename}` does not exist in job directory, or parsing it failed.
``` `job_test_unknown_fmt` is used in case no test file (produced by `bot/check-test.sh` provided by target repository) was found. diff --git a/app.cfg.example b/app.cfg.example index 7f4f40ee..ce34bc23 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -214,5 +214,5 @@ missing_modules = Slurm output lacks message "No missing modules!". no_tarball_message = Slurm output lacks message about created tarball. no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir. multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected. -job_result_unknown_fmt =
:shrug: UNKNOWN _(click triangle for detailed information)_
  • Job results file `{filename}` does not exist in job directory or reading it failed.
  • No artefacts were found/reported.
-job_test_unknown_fmt =
:shrug: UNKNOWN _(click triangle for detailed information)_
  • Job test file `{filename}` does not exist in job directory or reading it failed.
+job_result_unknown_fmt =
:shrug: UNKNOWN _(click triangle for detailed information)_
  • Job results file `{filename}` does not exist in job directory, or parsing it failed.
  • No artefacts were found/reported.
+job_test_unknown_fmt =
:shrug: UNKNOWN _(click triangle for detailed information)_
  • Job test file `{filename}` does not exist in job directory, or parsing it failed.
From d9547ad5eadcf1ed39a518afba62850fbaa6cd93 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 26 Nov 2023 15:00:59 +0100 Subject: [PATCH 8/9] use 'test result' rather than 'testing' in status table in PR comment --- eessi_bot_job_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_bot_job_manager.py b/eessi_bot_job_manager.py index afe41a5f..4830c1e3 100644 --- a/eessi_bot_job_manager.py +++ b/eessi_bot_job_manager.py @@ -585,7 +585,7 @@ def process_finished_job(self, finished_job): dt = datetime.now(timezone.utc) - comment_update += f"\n|{dt.strftime('%b %d %X %Z %Y')}|tested|" + comment_update += f"\n|{dt.strftime('%b %d %X %Z %Y')}|test result|" comment_update += f"{comment_description}|" # obtain id of PR comment to be updated (from file '_bot_jobID.metadata') From f94a922092c7521527e651d367df4d01affd3205 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 26 Nov 2023 15:01:42 +0100 Subject: [PATCH 9/9] use 'parsing' instead of 'reading' in README for `+job_result_unknown_fmt` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96d9753f..2fbd8339 100644 --- a/README.md +++ b/README.md @@ -609,7 +609,7 @@ multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching ` `multiple_tarballs` is used to report that multiple tarballs have been found. ``` -job_result_unknown_fmt =
:shrug: UNKNOWN _(click triangle for details)_
  • Job results file `{filename}` does not exist in job directory or reading it failed.
  • No artefacts were found/reported.
+job_result_unknown_fmt =
:shrug: UNKNOWN _(click triangle for details)_
  • Job results file `{filename}` does not exist in job directory, or parsing it failed.
  • No artefacts were found/reported.
``` `job_result_unknown_fmt` is used in case no result file (produced by `bot/check-build.sh` provided by target repository) was found.