Merge pull request #222 from trz42/feature/support_test_suite

add support to run test suite at the end of a build job

README.md

@@ -646,11 +646,17 @@ multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `
 `multiple_tarballs` is used to report that multiple tarballs have been found.
 
 ```
-job_result_unknown_fmt = <details><summary>:shrug: UNKNOWN _(click triangle for details)_</summary><ul><li>Job results file `{filename}` does not exist in job directory or reading it failed.</li><li>No artefacts were found/reported.</li></ul></details>
+job_result_unknown_fmt = <details><summary>:shrug: UNKNOWN _(click triangle for details)_</summary><ul><li>Job results file `{filename}` does not exist in job directory, or parsing it failed.</li><li>No artefacts were found/reported.</li></ul></details>
 ```
 `job_result_unknown_fmt` is used in case no result file (produced by `bot/check-build.sh`
 provided by target repository) was found.
 
+```
+job_test_unknown_fmt = <details><summary>:shrug: UNKNOWN _(click triangle for details)_</summary><ul><li>Job test file `{filename}` does not exist in job directory, or parsing it failed.</li></ul></details>
+```
+`job_test_unknown_fmt` is used in case no test file (produced by `bot/check-test.sh`
+provided by target repository) was found.
+
 # Instructions to run the bot components
 
 The bot consists of three components:

app.cfg.example

@@ -217,4 +217,5 @@ missing_modules = Slurm output lacks message "No missing modules!".
 no_tarball_message = Slurm output lacks message about created tarball.
 no_matching_tarball = No tarball matching `{tarball_pattern}` found in job dir.
 multiple_tarballs = Found {num_tarballs} tarballs in job dir - only 1 matching `{tarball_pattern}` expected.
-job_result_unknown_fmt = <details><summary>:shrug: UNKNOWN _(click triangle for detailed information)_<summary/><ul><li>Job results file `{filename}` does not exist in job directory or reading it failed.</li><li>No artefacts were found/reported.</li></ul></details>
+job_result_unknown_fmt = <details><summary>:shrug: UNKNOWN _(click triangle for detailed information)_</summary><ul><li>Job results file `{filename}` does not exist in job directory, or parsing it failed.</li><li>No artefacts were found/reported.</li></ul></details>
+job_test_unknown_fmt = <details><summary>:shrug: UNKNOWN _(click triangle for detailed information)_</summary><ul><li>Job test file `{filename}` does not exist in job directory, or parsing it failed.</li></ul></details>

eessi_bot_job_manager.py

@@ -51,6 +51,8 @@
 FINISHED_JOB_COMMENTS = "finished_job_comments"
 JOB_RESULT_COMMENT_DESCRIPTION = "comment_description"
 JOB_RESULT_UNKNOWN_FMT = "job_result_unknown_fmt"
+JOB_TEST_COMMENT_DESCRIPTION = "comment_description"
+JOB_TEST_UNKNOWN_FMT = "job_test_unknown_fmt"
 MISSING_MODULES = "missing_modules"
 MULTIPLE_TARBALLS = "multiple_tarballs"
 NEW_JOB_COMMENTS = "new_job_comments"
@@ -285,6 +287,24 @@ def read_job_result(self, job_result_file_path):
         else:
             return None
 
+    def read_job_test(self, job_test_file_path):
+        """
+        Read job test file and return the contents of the 'TEST' section.
+
+        Args:
+            job_test_file_path (string): path to job test file
+
+        Returns:
+            (ConfigParser): instance of ConfigParser corresponding to the
+                'TEST' section or None
+        """
+        # reuse function from module tools.job_metadata to read metadata file
+        test = read_metadata_file(job_test_file_path, self.logfile)
+        if test and "TEST" in test:
+            return test["TEST"]
+        else:
+            return None
+
     def process_new_job(self, new_job):
         """
         Process a new job by verifying that it is a bot job and if so
@@ -470,7 +490,8 @@ def process_finished_job(self, finished_job):
         """
         Process a finished job by
         - moving the symlink to the directory storing finished jobs,
-        - updating the PR comment with information from '*.result' file
+        - updating the PR comment with information from '*.result' and '*.test'
+          files
 
         Args:
             finished_job (dict): dictionary with information about the job
@@ -497,14 +518,21 @@ def process_finished_job(self, finished_job):
         os.rename(old_symlink, new_symlink)
 
         # REPORT status (to logfile in any case, to PR comment if accessible)
-        #   rely fully on what bot/check-build.sh has returned
-        #   check if file _bot_jobJOBID.result exists --> if so read it and
-        #   update PR comment
-        # contents of *.result file (here we only use section [RESULT])
-        #   [RESULT]
-        #   comment_description = _FULLY_DEFINED_UPDATE_TO_PR_COMMENT_
-        #   status = {SUCCESS,FAILURE,UNKNOWN}
-        #   artefacts = _LIST_OF_ARTEFACTS_TO_BE_DEPLOYED_
+        #  - rely fully on what bot/check-build.sh and bot/check-test.sh have
+        #    returned
+        #  - check if file _bot_jobJOBID.result exists --> if so read it and
+        #    update PR comment
+        #    . contents of *.result file (here we only use section [RESULT])
+        #      [RESULT]
+        #      comment_description = _FULLY_DEFINED_UPDATE_TO_PR_COMMENT_
+        #      status = {SUCCESS,FAILURE,UNKNOWN}
+        #      artefacts = _LIST_OF_ARTEFACTS_TO_BE_DEPLOYED_
+        #  - check if file _bot_jobJOBID.test exists --> if so read it and
+        #    update PR comment
+        #    . contents of *.test file (here we only use section [TEST])
+        #      [TEST]
+        #      comment_description = _FULLY_DEFINED_UPDATE_TO_PR_COMMENT_
+        #      status = {SUCCESS,FAILURE,UNKNOWN}
 
         # obtain format templates from app.cfg
         finished_job_comments_cfg = config.read_config()[FINISHED_JOB_COMMENTS]
@@ -533,6 +561,33 @@ def process_finished_job(self, finished_job):
         comment_update = f"\n|{dt.strftime('%b %d %X %Z %Y')}|finished|"
         comment_update += f"{comment_description}|"
 
+        # check if _bot_jobJOBID.test exits
+        # TODO if not found, assume test was not run (or failed, or ...) and add
+        # a message noting that ('not tested' + 'test suite not run or failed')
+        # --> bot/test.sh and bot/check-test.sh scripts are run in job script used by bot for 'build' action
+        job_test_file = f"_bot_job{job_id}.test"
+        job_test_file_path = os.path.join(new_symlink, job_test_file)
+        job_tests = self.read_job_test(job_test_file_path)
+
+        job_test_unknown_fmt = finished_job_comments_cfg[JOB_TEST_UNKNOWN_FMT]
+        # set fallback comment_description in case no test file was found
+        # (self.read_job_result returned None)
+        comment_description = job_test_unknown_fmt.format(filename=job_test_file)
+        if job_tests:
+            # get preformatted comment_description or use previously set default for unknown
+            comment_description = job_tests.get(JOB_TEST_COMMENT_DESCRIPTION, comment_description)
+
+        # report to log
+        log(f"{fn}(): finished job {job_id}, test suite result\n"
+            f"########\n"
+            f"comment_description: {comment_description}\n"
+            f"########\n", self.logfile)
+
+        dt = datetime.now(timezone.utc)
+
+        comment_update += f"\n|{dt.strftime('%b %d %X %Z %Y')}|test result|"
+        comment_update += f"{comment_description}|"
+
         # obtain id of PR comment to be updated (from file '_bot_jobID.metadata')
         metadata_file = f"_bot_job{job_id}.metadata"
         job_metadata_path = os.path.join(new_symlink, metadata_file)

scripts/bot-build.slurm

@@ -46,4 +46,27 @@ status = UNKNOWN
 artefacts =
 EOF
 fi
-echo "check result step finished"
+echo "check build step finished"
+TEST_SCRIPT=bot/test.sh
+if [ -f ${TEST_SCRIPT} ]; then
+    echo "${TEST_SCRIPT} script found in '${PWD}', so running it!"
+    ${TEST_SCRIPT}
+    echo "${TEST_SCRIPT} finished"
+else
+    echo "could not find ${TEST_SCRIPT} script in '${PWD}'" >&2
+fi
+CHECK_TEST_SCRIPT=bot/check-test.sh
+if [ -f ${CHECK_TEST_SCRIPT} ]; then
+    echo "${CHECK_TEST_SCRIPT} script found in '${PWD}', so running it!"
+    ${CHECK_TEST_SCRIPT}
+else
+    echo "could not find ${CHECK_TEST_SCRIPT} script in '${PWD}' ..."
+    echo "... depositing default _bot_job${SLURM_JOB_ID}.test file in '${PWD}'"
+    cat << 'EOF' > _bot_job${SLURM_JOB_ID}.test
+[RESULT]
+comment_description = <details><summary>:shrug: UNKNOWN _(click triangle for detailed information)_<summary/><ul><li>Did not find `bot/check-test.sh` script in job's work directory.</li><li>*Check job manually or ask an admin of the bot instance to assist you.*</li></ul></details>
+status = UNKNOWN
+artefacts =
+EOF
+fi
+echo "check test step finished"