diff --git a/MC/bin/o2_dpg_workflow_runner.py b/MC/bin/o2_dpg_workflow_runner.py index c9e76fcc9..7765189b0 100755 --- a/MC/bin/o2_dpg_workflow_runner.py +++ b/MC/bin/o2_dpg_workflow_runner.py @@ -667,6 +667,7 @@ def monitor(self, process_list): def waitforany(self, process_list, finished): failuredetected = False failingpids = [] + failingtasks = [] if len(process_list)==0: return False @@ -690,15 +691,32 @@ def waitforany(self, process_list, finished): if returncode!=0: failuredetected = True failingpids.append(pid) + failingtasks.append(p[0]) if failuredetected and self.stoponfailure: actionlogger.info('Stoping pipeline due to failure in stages with PID ' + str(failingpids)) # self.analyse_files_and_connections() + self.cat_logfiles_tostdout(failingtasks) + self.stop_pipeline_and_exit(process_list) # empty finished means we have to wait more return len(finished)==0 + def cat_logfiles_tostdout(self, taskids): + # In case of errors we can cat the logfiles for this taskname + # to stdout. Assuming convention that "taskname" translates to "taskname.log" logfile. + for tid in taskids: + taskspec = self.workflowspec['stages'][tid] + taskname = taskspec['name'] + filename = taskname + '.log' + directory = taskspec['cwd'] + path = directory + '/' + filename + if os.path.exists(path): + print (' ----> START OF LOGFILE ', path, ' -----') + os.system('cat ' + path) + print (' <---- END OF LOGFILE ', path, ' -----') + def analyse_files_and_connections(self): for p,s in self.pid_to_files.items(): for f in s: @@ -906,7 +924,10 @@ def execute(self): parser.add_argument('--mem-limit', help='Set memory limit as scheduling constraint', default=max_system_mem) parser.add_argument('--cpu-limit', help='Set CPU limit (core count)', default=8) -parser.add_argument('--cgroup', help='Execute pipeline under a given cgroup (e.g., 8coregrid) emulating resource constraints. This must exist and the tasks file must be writable to with the current user.') +parser.add_argument('--cgroup', help='Execute pipeline under a given cgroup (e.g., 8coregrid) emulating resource constraints. This m\ +ust exist and the tasks file must be writable to with the current user.') +parser.add_argument('--stdout-on-failure', action='store_true', help='Print log files of failing tasks to stdout,') + args = parser.parse_args() print (args)