diff --git a/README.md b/README.md index 8af58a9..ebf4e29 100644 --- a/README.md +++ b/README.md @@ -645,17 +645,24 @@ scontrol_command = /usr/bin/scontrol #### `[submitted_job_comments]` section The `[submitted_job_comments]` section specifies templates for messages about newly submitted jobs. -``` -initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}` -``` -`initial_comment` is used to create a comment to a PR when a new job has been created. - ``` awaits_release = job id `{job_id}` awaits release by job manager ``` `awaits_release` is used to provide a status update of a job (shown as a row in the job's status table). +``` +initial_comment = New job on instance `{app_name}` for architecture `{arch_name}`{accelerator_spec} for repository `{repo_id}` in job dir `{symlink}` +``` +`initial_comment` is used to create a comment to a PR when a new job has been +created. Note, the part '{accelerator_spec}' is only filled-in by the bot if the +argument 'accelerator' to the `bot: build` command have been used. +``` +with_accelerator = " and accelerator {accelerator}" +``` +`with_accelerator` is used to provide information about the accelerator the job +should build for if and only if the argument `accelerator:X/Y` has been provided. + #### `[new_job_comments]` section The `[new_job_comments]` section sets templates for messages about jobs whose `hold` flag was released. diff --git a/app.cfg.example b/app.cfg.example index 25abf91..2e50834 100644 --- a/app.cfg.example +++ b/app.cfg.example @@ -243,8 +243,9 @@ scontrol_command = /usr/bin/scontrol # are removed, the output (in PR comments) will lack important # information. [submitted_job_comments] -initial_comment = New job on instance `{app_name}` for architecture `{arch_name}` for repository `{repo_id}` in job dir `{symlink}` awaits_release = job id `{job_id}` awaits release by job manager +initial_comment = New job on instance `{app_name}` for CPU micro-architecture `{arch_name}`{accelerator_spec} for repository `{repo_id}` in job dir `{symlink}` +with_accelerator = " and accelerator {accelerator}" [new_job_comments] diff --git a/tasks/build.py b/tasks/build.py index a967ef7..96d04fe 100644 --- a/tasks/build.py +++ b/tasks/build.py @@ -33,6 +33,7 @@ # Local application imports (anything from EESSI/eessi-bot-software-layer) from connections import github from tools import config, cvmfs_repository, job_metadata, pr_comments, run_cmd +import tools.filter as tools_filter # defaults (used if not specified via, eg, 'app.cfg') @@ -46,7 +47,7 @@ _ERROR_NONE = "none" -Job = namedtuple('Job', ('working_dir', 'arch_target', 'repo_id', 'slurm_opts', 'year_month', 'pr_id')) +Job = namedtuple('Job', ('working_dir', 'arch_target', 'repo_id', 'slurm_opts', 'year_month', 'pr_id', 'accelerator')) # global repo_cfg repo_cfg = {} @@ -474,6 +475,12 @@ def prepare_jobs(pr, cfg, event_info, action_filter): # call to just before download_pr year_month, pr_id, run_dir = create_pr_dir(pr, cfg, event_info) + accelerator = "none" + # determine accelerator from action_filter argument + accelerators = action_filter.get_filter_by_component(tools_filter.FILTER_COMPONENT_ACCEL) + if len(accelerators) > 0: + accelerator = accelerators[0] + jobs = [] for arch, slurm_opt in arch_map.items(): arch_dir = arch.replace('/', '_') @@ -501,6 +508,15 @@ def prepare_jobs(pr, cfg, event_info, action_filter): continue else: log(f"{fn}(): context DOES satisfy filter(s), going on with job") + # we reached this point when the filter matched (otherwise we + # 'continue' with the next repository) + # for each match of the filter we create a specific job directory + # however, matching CPU architectures works differently to handling + # accelerators; multiple CPU architectures defined in arch_target_map + # can match the (CPU) architecture component of a filter; in + # contrast, the value of the accelerator filter is just passed down + # to scripts in bot/ directory of the pull request (see function + # prepare_job_cfg and creation of Job tuple below) job_dir = os.path.join(run_dir, arch_dir, repo_id) os.makedirs(job_dir, exist_ok=True) log(f"{fn}(): job_dir '{job_dir}'") @@ -514,10 +530,12 @@ def prepare_jobs(pr, cfg, event_info, action_filter): cpu_target = '/'.join(arch.split('/')[1:]) os_type = arch.split('/')[0] log(f"{fn}(): arch = '{arch}' => cpu_target = '{cpu_target}' , os_type = '{os_type}'") - prepare_job_cfg(job_dir, build_env_cfg, repocfg, repo_id, cpu_target, os_type) + + log(f"{fn}(): accelerator = '{accelerator}'") + prepare_job_cfg(job_dir, build_env_cfg, repocfg, repo_id, cpu_target, os_type, accelerator) # enlist jobs to proceed - job = Job(job_dir, arch, repo_id, slurm_opt, year_month, pr_id) + job = Job(job_dir, arch, repo_id, slurm_opt, year_month, pr_id, accelerator) jobs.append(job) log(f"{fn}(): {len(jobs)} jobs to proceed after applying white list") @@ -527,7 +545,7 @@ def prepare_jobs(pr, cfg, event_info, action_filter): return jobs -def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir, os_type): +def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir, os_type, accelerator): """ Set up job configuration file 'job.cfg' in directory /cfg @@ -538,6 +556,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir, repo_id (string): identifier of the repository to build for software_subdir (string): software subdirectory to build for (e.g., 'x86_64/generic') os_type (string): type of the os (e.g., 'linux') + accelerator (string): defines accelerator to build for (e.g., 'nvidia/cc80') Returns: None (implicitly) @@ -563,6 +582,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir, # [architecture] # software_subdir = software_subdir # os_type = os_type + # accelerator = accelerator job_cfg = configparser.ConfigParser() job_cfg[job_metadata.JOB_CFG_SITE_CONFIG_SECTION] = {} build_env_to_job_cfg_keys = { @@ -607,6 +627,7 @@ def prepare_job_cfg(job_dir, build_env_cfg, repos_cfg, repo_id, software_subdir, job_cfg[job_cfg_arch_section] = {} job_cfg[job_cfg_arch_section][job_metadata.JOB_CFG_ARCHITECTURE_SOFTWARE_SUBDIR] = software_subdir job_cfg[job_cfg_arch_section][job_metadata.JOB_CFG_ARCHITECTURE_OS_TYPE] = os_type + job_cfg[job_cfg_arch_section][job_metadata.JOB_CFG_ARCHITECTURE_ACCELERATOR] = accelerator # copy contents of directory containing repository configuration to directory # containing job configuration/metadata @@ -726,11 +747,19 @@ def create_pr_comment(job, job_id, app_name, pr, gh, symlink): # obtain arch from job.arch_target which has the format OS/ARCH arch_name = '-'.join(job.arch_target.split('/')[1:]) + submitted_job_comments_cfg = config.read_config()[config.SECTION_SUBMITTED_JOB_COMMENTS] + + # obtain accelerator from job.accelerator + accelerator = job.accelerator + accelerator_spec_str = '' + if not accelerator is 'none': + accelerator_spec = f"{submitted_job_comments_cfg[config.SUBMITTED_JOB_COMMENTS_SETTING_WITH_ACCELERATOR]}" + accelerator_spec_str = accelerator_spec.format(accelerator=accelerator) + # get current date and time dt = datetime.now(timezone.utc) # construct initial job comment - submitted_job_comments_cfg = config.read_config()[config.SECTION_SUBMITTED_JOB_COMMENTS] job_comment = (f"{submitted_job_comments_cfg[config.SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT]}" f"\n|date|job status|comment|\n" f"|----------|----------|------------------------|\n" @@ -741,7 +770,8 @@ def create_pr_comment(job, job_id, app_name, pr, gh, symlink): arch_name=arch_name, symlink=symlink, repo_id=job.repo_id, - job_id=job_id) + job_id=job_id, + accelerator_spec=accelerator_spec_str) # create comment to pull request repo_name = pr.base.repo.full_name diff --git a/tools/config.py b/tools/config.py index da8a287..ff641eb 100644 --- a/tools/config.py +++ b/tools/config.py @@ -105,8 +105,9 @@ RUNNING_JOB_COMMENTS_SETTING_RUNNING_JOB = 'running_job' SECTION_SUBMITTED_JOB_COMMENTS = 'submitted_job_comments' -SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT = 'initial_comment' SUBMITTED_JOB_COMMENTS_SETTING_AWAITS_RELEASE = 'awaits_release' +SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT = 'initial_comment' +SUBMITTED_JOB_COMMENTS_SETTING_WITH_ACCELERATOR = 'with_accelerator' SECTION_CLEAN_UP = 'clean_up' CLEAN_UP_SETTING_TRASH_BIN_ROOT_DIR = 'trash_bin_dir' diff --git a/tools/job_metadata.py b/tools/job_metadata.py index 286f71c..d400019 100644 --- a/tools/job_metadata.py +++ b/tools/job_metadata.py @@ -33,6 +33,7 @@ JOB_CFG_ARCHITECTURE_SECTION = "architecture" JOB_CFG_ARCHITECTURE_OS_TYPE = "os_type" JOB_CFG_ARCHITECTURE_SOFTWARE_SUBDIR = "software_subdir" +JOB_CFG_ARCHITECTURE_ACCELERATOR = "accelerator" JOB_CFG_REPOSITORY_SECTION = "repository" JOB_CFG_REPOSITORY_CONTAINER = "container"