Write offending script to stderr (#i530)

.github/linters/.python-lint

@@ -105,6 +105,7 @@ disable=abstract-method,
       missing-module-docstring,
       no-member,
       no-name-in-module,
+      no-self-use,
       no-value-for-parameter,
       pointless-statement,
       protected-access,

src/sos/actions.py

@@ -411,10 +411,14 @@ def run(self, **kwargs):
             else:
                 raise RuntimeError(f"Unacceptable interpreter {self.interpreter}")
 
+            debug_script_path = os.path.dirname(os.path.abspath(kwargs["stderr"])) if ("stderr" in kwargs and kwargs["stderr"] is not False and
+                                os.path.isdir(os.path.dirname(os.path.abspath(kwargs["stderr"])))) else env.exec_dir
             debug_script_file = os.path.join(
-                env.exec_dir,
+                debug_script_path,
                 f'{env.sos_dict["step_name"]}_{env.sos_dict["_index"]}_{str(uuid.uuid4())[:8]}{self.suffix}',
             )
+            debug_script_msg = f'\n>>> START SCRIPT ({debug_script_file}) <<<\n\n{self.script.strip()}\n\n>>> END SCRIPT <<<\n'
+
             # with open(debug_script_file, 'w') as sfile:
             #    sfile.write(self.script)
             # env.log_to_file('ACTION', self.script)
@@ -523,6 +527,7 @@ def run(self, **kwargs):
 
                         ret = pexpect_run(cmd.strip())
                 elif "__std_out__" in env.sos_dict and "__std_err__" in env.sos_dict:
+                    # task execution
                     if "stdout" in kwargs or "stderr" in kwargs:
                         if "stdout" in kwargs:
                             if kwargs["stdout"] is False:
@@ -547,6 +552,9 @@ def run(self, **kwargs):
                         p = subprocess.Popen(cmd, shell=True, stderr=se, stdout=so)
                         ret = p.wait()
 
+                        if ret != 0:
+                            se.write(debug_script_msg.encode())
+
                         if so != subprocess.DEVNULL:
                             so.close()
                         if se != subprocess.DEVNULL:
@@ -557,6 +565,9 @@ def run(self, **kwargs):
                                                                                  "ab") as se:
                             p = subprocess.Popen(cmd, shell=True, stderr=se, stdout=so)
                             ret = p.wait()
+
+                            if ret != 0:
+                                se.write(debug_script_msg.encode())
                     else:
                         p = subprocess.Popen(
                             cmd,
@@ -565,6 +576,9 @@ def run(self, **kwargs):
                             stdout=subprocess.DEVNULL,
                         )
                         ret = p.wait()
+
+                        if ret != 0:
+                            sys.stderr.write(debug_script_msg)
                 else:
                     if "stdout" in kwargs:
                         if kwargs["stdout"] is False:
@@ -587,8 +601,14 @@ def run(self, **kwargs):
                         se = subprocess.DEVNULL
 
                     p = subprocess.Popen(cmd, shell=True, stderr=se, stdout=so)
-
                     ret = p.wait()
+
+                    if ret != 0:
+                        if se:
+                            se.write(debug_script_msg.encode())
+                        else:
+                            sys.stderr.write(debug_script_msg)
+
                     if so is not None and so != subprocess.DEVNULL:
                         so.close()
                     if se is not None and se != subprocess.DEVNULL:

src/sos/eval.py

@@ -508,4 +508,3 @@ def analyze_global_statements(global_stmt):
                     f"Variable {key} cannot be defined in global section because it cannot be pickled to workers."
                 ) from e
     return global_def, global_vars
-

src/sos/singularity/client.py

@@ -332,7 +332,7 @@ def run(self,
             for opt in ['nv', 'nvccli', 'disable_cache', 'nohttps', 'nonet', 'vm_err', 'writable',
                         'writable_tmpfs', 'vm', 'uts', 'userns', 'rocm', 'pid', 'passphrase',
                         'no_mark', 'no_privs', 'no_init', 'no_https', 'no_home', 'net',
-                        'keep_privs', 'fakeroot'', disable_cache', 'containall', 'contain',
+                        'keep_privs', 'fakeroot', 'disable_cache', 'containall', 'contain',
                         'compat', 'cleanenv', 'allow_setuid']:
                 if opt in kwargs and kwargs[opt]:
                     exec_opts.append('--' + opt.replace('_', '-'))

src/sos/task_engines.py

@@ -376,7 +376,8 @@ def submit_task(self, task_id):
         self.engine_ready.wait()
 
         # submit tasks simply add task_id to pending task list
-        with threading.Lock():
+        lock = threading.Lock()
+        with lock:
             # if already in
             # if task_id in self.running_tasks or task_id in self.pending_tasks:
             #    self.notify_controller('{} ``{}``'.format(task_id, self.task_status[task_id]))

src/sos/tasks.py

@@ -479,6 +479,8 @@ def add_outputs(self, keep_result=False):
         with fasteners.InterProcessLock(os.path.join(env.temp_dir, self.task_id + ".lck")):
             with open(self.task_file, "r+b") as fh:
                 header = self._read_header(fh)
+                result = ''
+                signature = ''
                 if header.result_size != 0:
                     if not keep_result:
                         result_size = 0
@@ -514,9 +516,9 @@ def add_outputs(self, keep_result=False):
                     fh.write(stdout)
                 if stderr:
                     fh.write(stderr)
-                if result_size > 0:
+                if result_size > 0 and result:
                     fh.write(result)
-                if signature_size > 0:
+                if signature_size > 0 and signature:
                     fh.write(signature)
 
     def add_result(self, result: dict = {}):