reorganisation of constants and validation of config settings

- string constants being used have been defined in the correct modules
  - when they are used they have to be prefixed with the module, hence
    it becomes more clear to which module they belong
- a new file tools/cvmfs_repository.py for constants used for the settings of a
  CernVM-FS repository is added
- both the event handler and job manager verify at the start that
  required/recommended settings are defined in 'app.cfg'

connections/github.py

@@ -46,10 +46,10 @@ def get_token():
 
     global _token
     cfg = config.read_config()
-    github_cfg = cfg['github']
-    app_id = github_cfg.get('app_id')
-    installation_id = github_cfg.get('installation_id')
-    private_key_path = github_cfg.get('private_key')
+    github_cfg = cfg[config.SECTION_GITHUB]
+    app_id = github_cfg.get(config.GITHUB_SETTING_APP_ID)
+    installation_id = github_cfg.get(config.GITHUB_SETTING_INSTALLATION_ID)
+    private_key_path = github_cfg.get(config.GITHUB_SETTING_PRIVATE_KEY)
     private_key = ''
 
     with open(private_key_path, 'r') as private_key_file:

eessi_bot_event_handler.py

@@ -26,10 +26,8 @@
 
 # Local application imports (anything from EESSI/eessi-bot-software-layer)
 from connections import github
-import tasks.build as build
 from tasks.build import check_build_permission, get_architecture_targets, get_repo_cfg, \
     request_bot_build_issue_comments, submit_build_jobs
-import tasks.deploy as deploy
 from tasks.deploy import deploy_built_artefacts
 from tools import config
 from tools.args import event_handler_parse
@@ -39,11 +37,59 @@
 from tools.pr_comments import create_comment
 
 
-APP_NAME = "app_name"
-BOT_CONTROL = "bot_control"
-COMMAND_RESPONSE_FMT = "command_response_fmt"
-GITHUB = "github"
-REPO_TARGET_MAP = "repo_target_map"
+REQUIRED_CONFIG = {
+    config.SECTION_ARCHITECTURETARGETS: [
+        config.ARCHITECTURETARGETS_SETTING_ARCH_TARGET_MAP],       # required
+    config.SECTION_BOT_CONTROL: [
+        config.BOT_CONTROL_SETTING_COMMAND_PERMISSION,             # required
+        config.BOT_CONTROL_SETTING_COMMAND_RESPONSE_FMT],          # required
+    config.SECTION_BUILDENV: [
+        config.BUILDENV_SETTING_BUILD_JOB_SCRIPT,                  # required
+        config.BUILDENV_SETTING_BUILD_LOGS_DIR,                    # optional+recommended
+        config.BUILDENV_SETTING_BUILD_PERMISSION,                  # optional+recommended
+        config.BUILDENV_SETTING_CONTAINER_CACHEDIR,                # optional+recommended
+        # config.BUILDENV_SETTING_CVMFS_CUSTOMIZATIONS,              # optional
+        # config.BUILDENV_SETTING_HTTPS_PROXY,                       # optional
+        # config.BUILDENV_SETTING_HTTP_PROXY,                        # optional
+        config.BUILDENV_SETTING_JOBS_BASE_DIR,                     # required
+        # config.BUILDENV_SETTING_LOAD_MODULES,                      # optional
+        config.BUILDENV_SETTING_LOCAL_TMP,                         # required
+        config.BUILDENV_SETTING_NO_BUILD_PERMISSION_COMMENT,       # required
+        config.BUILDENV_SETTING_SHARED_FS_PATH,                    # optional+recommended
+        # config.BUILDENV_SETTING_SLURM_PARAMS,                      # optional
+        config.BUILDENV_SETTING_SUBMIT_COMMAND],                   # required
+    config.SECTION_DEPLOYCFG: [
+        config.DEPLOYCFG_SETTING_ARTEFACT_PREFIX,                  # (required)
+        config.DEPLOYCFG_SETTING_ARTEFACT_UPLOAD_SCRIPT,           # required
+        config.DEPLOYCFG_SETTING_BUCKET_NAME,                      # required
+        config.DEPLOYCFG_SETTING_DEPLOY_PERMISSION,                # optional+recommended
+        # config.DEPLOYCFG_SETTING_ENDPOINT_URL,                     # optional
+        config.DEPLOYCFG_SETTING_METADATA_PREFIX,                  # (required)
+        config.DEPLOYCFG_SETTING_NO_DEPLOY_PERMISSION_COMMENT,     # required
+        config.DEPLOYCFG_SETTING_UPLOAD_POLICY],                   # required
+    config.SECTION_DOWNLOAD_PR_COMMENTS: [
+        config.DOWNLOAD_PR_COMMENTS_SETTING_CURL_FAILURE,          # required
+        config.DOWNLOAD_PR_COMMENTS_SETTING_CURL_TIP,              # required
+        config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_APPLY_FAILURE,     # required
+        config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_APPLY_TIP,         # required
+        config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CHECKOUT_FAILURE,  # required
+        config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CHECKOUT_TIP,      # required
+        config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_FAILURE,     # required
+        config.DOWNLOAD_PR_COMMENTS_SETTING_GIT_CLONE_TIP],        # required
+    config.SECTION_EVENT_HANDLER: [
+        config.EVENT_HANDLER_SETTING_LOG_PATH],                    # required
+    config.SECTION_GITHUB: [
+        config.GITHUB_SETTING_APP_ID,                              # required
+        config.GITHUB_SETTING_APP_NAME,                            # required
+        config.GITHUB_SETTING_INSTALLATION_ID,                     # required
+        config.GITHUB_SETTING_PRIVATE_KEY],                        # required
+    config.SECTION_REPO_TARGETS: [
+        config.REPO_TARGETS_SETTING_REPO_TARGET_MAP,               # required
+        config.REPO_TARGETS_SETTING_REPOS_CFG_DIR],                # required
+    config.SECTION_SUBMITTED_JOB_COMMENTS: [
+        config.SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT,     # required
+        config.SUBMITTED_JOB_COMMENTS_SETTING_AWAITS_RELEASE]      # required
+    }
 
 
 class EESSIBotSoftwareLayer(PyGHee):
@@ -62,8 +108,8 @@ def __init__(self, *args, **kwargs):
         super(EESSIBotSoftwareLayer, self).__init__(*args, **kwargs)
 
         self.cfg = config.read_config()
-        event_handler_cfg = self.cfg['event_handler']
-        self.logfile = event_handler_cfg.get('log_path')
+        event_handler_cfg = self.cfg[config.SECTION_EVENT_HANDLER]
+        self.logfile = event_handler_cfg.get(config.EVENT_HANDLER_SETTING_LOG_PATH)
 
     def log(self, msg, *args):
         """
@@ -110,8 +156,8 @@ def handle_issue_comment_event(self, event_info, log_file=None):
         #      log level is set to debug
         self.log(f"Comment in {issue_url} (owned by @{owner}) {action} by @{sender}")
 
-        app_name = self.cfg[GITHUB][APP_NAME]
-        command_response_fmt = self.cfg[BOT_CONTROL][COMMAND_RESPONSE_FMT]
+        app_name = self.cfg[config.SECTION_GITHUB][config.GITHUB_SETTING_APP_NAME]
+        command_response_fmt = self.cfg[config.SECTION_BOT_CONTROL][config.BOT_CONTROL_SETTING_COMMAND_RESPONSE_FMT]
 
         # currently, only commands in new comments are supported
         #  - commands have the syntax 'bot: COMMAND [ARGS*]'
@@ -301,8 +347,8 @@ def handle_pull_request_labeled_event(self, event_info, pr):
             request_body = event_info['raw_request_body']
             repo_name = request_body['repository']['full_name']
             pr_number = request_body['pull_request']['number']
-            app_name = self.cfg[GITHUB][APP_NAME]
-            command_response_fmt = self.cfg[BOT_CONTROL][COMMAND_RESPONSE_FMT]
+            app_name = self.cfg[config.SECTION_GITHUB][config.GITHUB_SETTING_APP_NAME]
+            command_response_fmt = self.cfg[config.SECTION_BOT_CONTROL][config.BOT_CONTROL_SETTING_COMMAND_RESPONSE_FMT]
             comment_body = command_response_fmt.format(
                 app_name=app_name,
                 comment_response=msg,
@@ -330,7 +376,7 @@ def handle_pull_request_opened_event(self, event_info, pr):
                 PyGithub, not the github from the internal connections module)
         """
         self.log("PR opened: waiting for label bot:build")
-        app_name = self.cfg[GITHUB][APP_NAME]
+        app_name = self.cfg[config.SECTION_GITHUB][config.GITHUB_SETTING_APP_NAME]
         # TODO check if PR already has a comment with arch targets and
         # repositories
         arch_map = get_architecture_targets(self.cfg)
@@ -343,7 +389,8 @@ def handle_pull_request_opened_event(self, event_info, pr):
             comment += f"{', '.join([f'`{arch}`' for arch in architectures])}"
         else:
             comment += "none"
-        repositories = list(set([repo_id for repo_ids in repo_cfg[REPO_TARGET_MAP].values() for repo_id in repo_ids]))
+        repositories = list(set([repo_id for repo_ids in repo_cfg[config.REPO_TARGETS_SETTING_REPO_TARGET_MAP].values()
+                            for repo_id in repo_ids]))
         comment += "\n- repositories: "
         if len(repositories) > 0:
             comment += f"{', '.join([f'`{repo_id}`' for repo_id in repositories])}"
@@ -547,9 +594,7 @@ def start(self, app, port=3000):
         print(port_info)
         self.log(port_info)
 
-        event_handler_cfg = self.cfg['event_handler']
-        my_logfile = event_handler_cfg.get('log_path')
-        log_file_info = "logging in to %s" % my_logfile
+        log_file_info = "logging in to %s" % self.logfile
         print(log_file_info)
         self.log(log_file_info)
         waitress.serve(app, listen='*:%s' % port)
@@ -563,13 +608,8 @@ def main():
     """
     opts = event_handler_parse()
 
-    required_config = {
-        build.SUBMITTED_JOB_COMMENTS: [build.INITIAL_COMMENT, build.AWAITS_RELEASE],
-        build.BUILDENV: [build.NO_BUILD_PERMISSION_COMMENT],
-        deploy.DEPLOYCFG: [deploy.NO_DEPLOY_PERMISSION_COMMENT]
-    }
     # config is read and checked for settings to raise an exception early when the event_handler starts.
-    config.check_required_cfg_settings(required_config)
+    config.check_required_cfg_settings(REQUIRED_CONFIG)
     github.connect()
 
     if opts.file:

eessi_bot_job_manager.py

@@ -48,21 +48,27 @@
 from tools.pr_comments import get_submitted_job_comment, update_comment
 
 
-AWAITS_LAUNCH = "awaits_launch"
-FINISHED_JOB_COMMENTS = "finished_job_comments"
-JOB_RESULT_COMMENT_DESCRIPTION = "comment_description"
-JOB_RESULT_UNKNOWN_FMT = "job_result_unknown_fmt"
-JOB_TEST_COMMENT_DESCRIPTION = "comment_description"
-JOB_TEST_UNKNOWN_FMT = "job_test_unknown_fmt"
-NEW_JOB_COMMENTS = "new_job_comments"
-RUNNING_JOB = "running_job"
-RUNNING_JOB_COMMENTS = "running_job_comments"
-
+# settings that are required in 'app.cfg'
 REQUIRED_CONFIG = {
-    FINISHED_JOB_COMMENTS: [JOB_RESULT_UNKNOWN_FMT, JOB_TEST_UNKNOWN_FMT],
-    NEW_JOB_COMMENTS: [AWAITS_LAUNCH],
-    RUNNING_JOB_COMMENTS: [RUNNING_JOB]
-}
+    config.SECTION_FINISHED_JOB_COMMENTS: [
+        config.FINISHED_JOB_COMMENTS_SETTING_JOB_RESULT_UNKNOWN_FMT,  # required
+        config.FINISHED_JOB_COMMENTS_SETTING_JOB_TEST_UNKNOWN_FMT],   # required
+    config.SECTION_GITHUB: [
+        config.GITHUB_SETTING_APP_ID,                                 # required
+        # config.GITHUB_SETTING_APP_NAME,                               # unused
+        config.GITHUB_SETTING_INSTALLATION_ID,                        # required
+        config.GITHUB_SETTING_PRIVATE_KEY],                           # required
+    config.SECTION_JOB_MANAGER: [
+        config.JOB_MANAGER_SETTING_LOG_PATH,                          # required
+        config.JOB_MANAGER_SETTING_JOB_IDS_DIR,                       # required
+        config.JOB_MANAGER_SETTING_POLL_COMMAND,                      # required
+        config.JOB_MANAGER_SETTING_POLL_INTERVAL,                     # optional+recommended
+        config.JOB_MANAGER_SETTING_SCONTROL_COMMAND],                 # required
+    config.SECTION_NEW_JOB_COMMENTS: [
+        config.NEW_JOB_COMMENTS_SETTING_AWAITS_LAUNCH],               # required
+    config.SECTION_RUNNING_JOB_COMMENTS: [
+        config.RUNNING_JOB_COMMENTS_SETTING_RUNNING_JOB]              # required
+    }
 
 
 class EESSIBotSoftwareLayerJobManager:
@@ -77,8 +83,8 @@ def __init__(self):
         configuration to set the path to the logfile.
         """
         cfg = config.read_config()
-        job_manager_cfg = cfg['job_manager']
-        self.logfile = job_manager_cfg.get('log_path')
+        job_manager_cfg = cfg[config.SECTION_JOB_MANAGER]
+        self.logfile = job_manager_cfg.get(config.JOB_MANAGER_SETTING_LOG_PATH)
 
     def get_current_jobs(self):
         """
@@ -340,10 +346,10 @@ def process_new_job(self, new_job):
 
             # update status table if we found a comment
             if "comment_id" in new_job:
-                new_job_comments_cfg = config.read_config()[NEW_JOB_COMMENTS]
+                new_job_comments_cfg = config.read_config()[config.SECTION_NEW_JOB_COMMENTS]
                 dt = datetime.now(timezone.utc)
                 update = "\n|%s|released|" % dt.strftime("%b %d %X %Z %Y")
-                update += f"{new_job_comments_cfg[AWAITS_LAUNCH]}|"
+                update += f"{new_job_comments_cfg[config.NEW_JOB_COMMENTS_SETTING_AWAITS_LAUNCH]}|"
                 update_comment(new_job["comment_id"], pr, update)
             else:
                 log(
@@ -412,8 +418,9 @@ def process_running_jobs(self, running_job):
 
         if "comment_id" in running_job:
             dt = datetime.now(timezone.utc)
-            running_job_comments_cfg = config.read_config()[RUNNING_JOB_COMMENTS]
-            running_msg = running_job_comments_cfg[RUNNING_JOB].format(job_id=running_job['jobid'])
+            running_job_comments_cfg = config.read_config()[config.SECTION_RUNNING_JOB_COMMENTS]
+            running_msg_fmt = running_job_comments_cfg[config.RUNNING_JOB_COMMENTS_SETTING_RUNNING_JOB]
+            running_msg = running_msg_fmt.format(job_id=running_job['jobid'])
             if "comment_body" in running_job and running_msg in running_job["comment_body"]:
                 log("Not updating comment, '%s' already found" % running_msg)
             else:
@@ -476,7 +483,7 @@ def process_finished_job(self, finished_job):
         #      status = {SUCCESS,FAILURE,UNKNOWN}
 
         # obtain format templates from app.cfg
-        finished_job_comments_cfg = config.read_config()[FINISHED_JOB_COMMENTS]
+        finished_job_comments_cfg = config.read_config()[config.SECTION_FINISHED_JOB_COMMENTS]
 
         # check if _bot_jobJOBID.result exits
         job_result_file = f"_bot_job{job_id}.result"
@@ -485,13 +492,13 @@ def process_finished_job(self, finished_job):
                                                          job_metadata.JOB_RESULT_SECTION,
                                                          self.logfile)
 
-        job_result_unknown_fmt = finished_job_comments_cfg[JOB_RESULT_UNKNOWN_FMT]
+        job_result_unknown_fmt = finished_job_comments_cfg[config.FINISHED_JOB_COMMENTS_SETTING_JOB_RESULT_UNKNOWN_FMT]
         # set fallback comment_description in case no result file was found
         # (job_metadata.get_section_from_file returned None)
         comment_description = job_result_unknown_fmt.format(filename=job_result_file)
         if job_results:
             # get preformatted comment_description or use previously set default for unknown
-            comment_description = job_results.get(JOB_RESULT_COMMENT_DESCRIPTION, comment_description)
+            comment_description = job_results.get(job_metadata.JOB_RESULT_COMMENT_DESCRIPTION, comment_description)
 
         # report to log
         log(f"{fn}(): finished job {job_id}\n"
@@ -514,13 +521,13 @@ def process_finished_job(self, finished_job):
                                                        job_metadata.JOB_TEST_SECTION,
                                                        self.logfile)
 
-        job_test_unknown_fmt = finished_job_comments_cfg[JOB_TEST_UNKNOWN_FMT]
+        job_test_unknown_fmt = finished_job_comments_cfg[config.FINISHED_JOB_COMMENTS_SETTING_JOB_TEST_UNKNOWN_FMT]
         # set fallback comment_description in case no test file was found
         # (job_metadata.get_section_from_file returned None)
         comment_description = job_test_unknown_fmt.format(filename=job_test_file)
         if job_tests:
             # get preformatted comment_description or use previously set default for unknown
-            comment_description = job_tests.get(JOB_TEST_COMMENT_DESCRIPTION, comment_description)
+            comment_description = job_tests.get(job_metadata.JOB_TEST_COMMENT_DESCRIPTION, comment_description)
 
         # report to log
         log(f"{fn}(): finished job {job_id}, test suite result\n"
@@ -611,16 +618,16 @@ def main():
     job_manager.scontrol_command = ""
     if max_iter != 0:
         cfg = config.read_config()
-        job_mgr = cfg["job_manager"]
-        job_manager.job_ids_dir = job_mgr.get("job_ids_dir")
+        job_mgr = cfg[config.SECTION_JOB_MANAGER]
+        job_manager.job_ids_dir = job_mgr.get(config.JOB_MANAGER_SETTING_JOB_IDS_DIR)
         job_manager.submitted_jobs_dir = os.path.join(
             job_manager.job_ids_dir, "submitted"
         )
-        job_manager.poll_command = job_mgr.get("poll_command") or False
-        poll_interval = int(job_mgr.get("poll_interval") or 0)
+        job_manager.poll_command = job_mgr.get(config.JOB_MANAGER_SETTING_POLL_COMMAND) or False
+        poll_interval = int(job_mgr.get(config.JOB_MANAGER_SETTING_POLL_INTERVAL) or 0)
         if poll_interval <= 0:
             poll_interval = 60
-        job_manager.scontrol_command = job_mgr.get("scontrol_command") or False
+        job_manager.scontrol_command = job_mgr.get(config.JOB_MANAGER_SETTING_SCONTROL_COMMAND) or False
         os.makedirs(job_manager.submitted_jobs_dir, exist_ok=True)
 
     # max_iter