From 1483d422a3f366527f853d77256765eed98d082e Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 18 Sep 2023 13:29:23 -0500 Subject: [PATCH 01/22] Add shared filtering --- jarvis_util/introspect/system_info.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jarvis_util/introspect/system_info.py b/jarvis_util/introspect/system_info.py index bf2c15a..90c4aa0 100644 --- a/jarvis_util/introspect/system_info.py +++ b/jarvis_util/introspect/system_info.py @@ -183,7 +183,7 @@ def __init__(self, exec_info): def wait(self): super().wait() total = [] - for host, stdout in self.stdout.items(): + for host, stdout in self.stdout.items():F lsblk_data = yaml.load(stdout, Loader=yaml.FullLoader) for dev in lsblk_data: if dev['tran'] == 'pcie': @@ -775,6 +775,7 @@ def find_storage(self, min_cap=None, min_avail=None, mount_res=None, + shared=None, df=None): """ Find a set of storage devices. @@ -790,6 +791,7 @@ def find_storage(self, :param min_cap: Remove devices with too little overall capacity :param min_avail: Remove devices with too little available space :param mount_res: A regex or list of regexes to match mount points + :param shared: Whether to search for devices which are shared :param df: The data frame to run this query :return: Dataframe """ @@ -829,6 +831,8 @@ def find_storage(self, if common and condense: df = df.groupby(['mount']).first().reset_index() # df = df.drop_columns('host') + if shared is not None: + df = df[lambda r: r['shared'] == shared] return df @staticmethod From f89c8ea136892c617a8c843762e79a7de97dfab2 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 18 Sep 2023 13:49:56 -0500 Subject: [PATCH 02/22] Fix unexpected indentation --- jarvis_util/introspect/system_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jarvis_util/introspect/system_info.py b/jarvis_util/introspect/system_info.py index 90c4aa0..c26fe46 100644 --- a/jarvis_util/introspect/system_info.py +++ b/jarvis_util/introspect/system_info.py @@ -183,7 +183,7 @@ def __init__(self, exec_info): def wait(self): super().wait() total = [] - for host, stdout in self.stdout.items():F + for host, stdout in self.stdout.items(): lsblk_data = yaml.load(stdout, Loader=yaml.FullLoader) for dev in lsblk_data: if dev['tran'] == 'pcie': From 6a79c8a64db864574560dca2e8f2459ddfcb6bdb Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 07:13:20 -0500 Subject: [PATCH 03/22] Debug scp --- jarvis_util/jutil_manager.py | 1 + jarvis_util/shell/scp.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/jarvis_util/jutil_manager.py b/jarvis_util/jutil_manager.py index c229b24..abeb214 100644 --- a/jarvis_util/jutil_manager.py +++ b/jarvis_util/jutil_manager.py @@ -25,4 +25,5 @@ def __init__(self): self.hide_output = False self.debug_mpi_exec = False self.debug_local_exec = False + self.debug_scp = False diff --git a/jarvis_util/shell/scp.py b/jarvis_util/shell/scp.py index 8b39301..fcca86d 100644 --- a/jarvis_util/shell/scp.py +++ b/jarvis_util/shell/scp.py @@ -47,6 +47,8 @@ def rsync_cmd(self, src_path, dst_path): else: lines.append(f'{self.addr}:{dst_path}') rsync_cmd = ' '.join(lines) + if self.debug_scp: + print(rsync_cmd) return rsync_cmd From 1282eecffece55125373e17c2ba0734e955c44bb Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 07:13:59 -0500 Subject: [PATCH 04/22] Debug scp --- jarvis_util/shell/scp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jarvis_util/shell/scp.py b/jarvis_util/shell/scp.py index fcca86d..ebdd4c6 100644 --- a/jarvis_util/shell/scp.py +++ b/jarvis_util/shell/scp.py @@ -47,7 +47,7 @@ def rsync_cmd(self, src_path, dst_path): else: lines.append(f'{self.addr}:{dst_path}') rsync_cmd = ' '.join(lines) - if self.debug_scp: + if self.jutil.debug_scp: print(rsync_cmd) return rsync_cmd From 595f7b4d91d8ac181656f33ec4bd7d53dae1d0ed Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 07:16:09 -0500 Subject: [PATCH 05/22] Add jutil to Executable --- jarvis_util/shell/exec_info.py | 2 ++ jarvis_util/shell/local_exec.py | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/jarvis_util/shell/exec_info.py b/jarvis_util/shell/exec_info.py index a415bc1..cf7b2a2 100644 --- a/jarvis_util/shell/exec_info.py +++ b/jarvis_util/shell/exec_info.py @@ -6,6 +6,7 @@ from enum import Enum from jarvis_util.util.hostfile import Hostfile +from jarvis_util.jutil_manager import JutilManager import os from abc import ABC, abstractmethod @@ -152,6 +153,7 @@ def __init__(self): self.exit_code = None self.stdout = '' self.stderr = '' + self.jutil = JutilManager.get_instance() def failed(self): return self.exit_code != 0 diff --git a/jarvis_util/shell/local_exec.py b/jarvis_util/shell/local_exec.py index c9eac96..8f02785 100644 --- a/jarvis_util/shell/local_exec.py +++ b/jarvis_util/shell/local_exec.py @@ -27,7 +27,6 @@ def __init__(self, cmd, exec_info): """ super().__init__() - jutil = JutilManager.get_instance() cmd = self.smash_cmd(cmd) # Managing console output and collection @@ -39,13 +38,13 @@ def __init__(self, cmd, exec_info): self.hide_output = exec_info.hide_output # pylint: disable=R1732 if self.collect_output is None: - self.collect_output = jutil.collect_output + self.collect_output = self.jutil.collect_output if self.pipe_stdout is not None: self.pipe_stdout_fp = open(self.pipe_stdout, 'wb') if self.pipe_stderr is not None: self.pipe_stderr_fp = open(self.pipe_stderr, 'wb') if self.hide_output is None: - self.hide_output = jutil.hide_output + self.hide_output = self.jutil.hide_output # pylint: enable=R1732 self.stdout = io.StringIO() self.stderr = io.StringIO() @@ -72,7 +71,7 @@ def __init__(self, cmd, exec_info): self.cwd = os.getcwd() else: self.cwd = exec_info.cwd - if jutil.debug_local_exec: + if self.jutil.debug_local_exec: print(cmd) self._start_bash_processes() From 71a366b9eb11523330af243d66d8ac16d9922595 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 07:21:09 -0500 Subject: [PATCH 06/22] Exec called before get_instance --- jarvis_util/shell/scp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jarvis_util/shell/scp.py b/jarvis_util/shell/scp.py index ebdd4c6..58b6a24 100644 --- a/jarvis_util/shell/scp.py +++ b/jarvis_util/shell/scp.py @@ -28,6 +28,7 @@ def __init__(self, src_path, dst_path, exec_info): self.pkey = exec_info.pkey self.port = exec_info.port self.sudo = exec_info.sudo + self.jutil = JutilManager.get_instance() super().__init__(self.rsync_cmd(src_path, dst_path), exec_info.mod(env=exec_info.basic_env)) From e63f5fee42afb9b105cb4adcfbee81554a1b4bfb Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 07:21:36 -0500 Subject: [PATCH 07/22] Exec called before get_instance --- jarvis_util/shell/scp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jarvis_util/shell/scp.py b/jarvis_util/shell/scp.py index 58b6a24..6530b99 100644 --- a/jarvis_util/shell/scp.py +++ b/jarvis_util/shell/scp.py @@ -4,6 +4,7 @@ """ from .local_exec import LocalExec from .exec_info import Executable +from jarvis_util.jutil_manager import JutilManager class _Scp(LocalExec): From b83e211c6c86affe5e491770501b1a4dabcf9848 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 07:22:58 -0500 Subject: [PATCH 08/22] Use IP instead of host name --- jarvis_util/shell/scp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jarvis_util/shell/scp.py b/jarvis_util/shell/scp.py index 6530b99..f1ed78f 100644 --- a/jarvis_util/shell/scp.py +++ b/jarvis_util/shell/scp.py @@ -22,7 +22,7 @@ def __init__(self, src_path, dst_path, exec_info): :param exec_info: Info needed to execute command with SSH """ - self.addr = exec_info.hostfile.hosts[0] + self.addr = exec_info.hostfile.hosts_ip[0] self.src_path = src_path self.dst_path = dst_path self.user = exec_info.user From bc63102a85e28bfc3518c7229fd256469b97db06 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 07:24:06 -0500 Subject: [PATCH 09/22] Add localhost check to rsync --- jarvis_util/shell/scp.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jarvis_util/shell/scp.py b/jarvis_util/shell/scp.py index f1ed78f..5e1df07 100644 --- a/jarvis_util/shell/scp.py +++ b/jarvis_util/shell/scp.py @@ -22,7 +22,9 @@ def __init__(self, src_path, dst_path, exec_info): :param exec_info: Info needed to execute command with SSH """ - self.addr = exec_info.hostfile.hosts_ip[0] + self.addr = exec_info.hostfile.hosts[0] + if self.addr == 'localhost' or self.addr == '127.0.0.1': + return self.src_path = src_path self.dst_path = dst_path self.user = exec_info.user From 4f5733a14d78c9ab67bce9512f3c03ce933d54d3 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 07:54:48 -0500 Subject: [PATCH 10/22] Enumerate host ips --- jarvis_util/util/hostfile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jarvis_util/util/hostfile.py b/jarvis_util/util/hostfile.py index 124d123..be4a105 100644 --- a/jarvis_util/util/hostfile.py +++ b/jarvis_util/util/hostfile.py @@ -202,6 +202,9 @@ def hostname_list(self): def enumerate(self): return enumerate(self.hosts) + def enumerate_ips(self): + return enumerate(self.hosts_ip) + def host_str(self, sep=','): return sep.join(self.hosts) From 37bbfa034fdfec24c591ebb8075ce034e4babfed Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 08:20:15 -0500 Subject: [PATCH 11/22] Make it so list returns a set of hostsfile classes instead of strings --- jarvis_util/util/hostfile.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/jarvis_util/util/hostfile.py b/jarvis_util/util/hostfile.py index be4a105..03feb99 100644 --- a/jarvis_util/util/hostfile.py +++ b/jarvis_util/util/hostfile.py @@ -193,17 +193,11 @@ def save(self, path): fp.write('\n'.join(self.all_hosts)) return self - def ip_list(self): - return self.hosts_ip - - def hostname_list(self): - return self.hosts + def list(self): + return [Hostfile(all_hosts=host) for host in self.hosts] def enumerate(self): - return enumerate(self.hosts) - - def enumerate_ips(self): - return enumerate(self.hosts_ip) + return enumerate(subhosts()) def host_str(self, sep=','): return sep.join(self.hosts) From 7dc9c753531271fcd623137800dbc5f1ddca6569 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 08:22:44 -0500 Subject: [PATCH 12/22] Make it so list returns a set of hostsfile classes instead of strings --- jarvis_util/util/hostfile.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jarvis_util/util/hostfile.py b/jarvis_util/util/hostfile.py index 03feb99..373c225 100644 --- a/jarvis_util/util/hostfile.py +++ b/jarvis_util/util/hostfile.py @@ -28,6 +28,7 @@ def __init__(self, hostfile=None, all_hosts=None, all_hosts_ip=None, self.hosts = [] self.all_hosts = [] self.all_hosts_ip = [] + self.subhosts = [] self.path = hostfile self.find_ips = find_ips @@ -53,6 +54,8 @@ def __init__(self, hostfile=None, all_hosts=None, all_hosts_ip=None, else: self._set_hosts(['localhost']) + self.hosts_iter = [Hostfile(hosts=host) for host in self.hosts] + def _load_hostfile(self, path): """ Expand a hostfile @@ -194,10 +197,10 @@ def save(self, path): return self def list(self): - return [Hostfile(all_hosts=host) for host in self.hosts] + return self.hosts_iter def enumerate(self): - return enumerate(subhosts()) + return enumerate(self.list()) def host_str(self, sep=','): return sep.join(self.hosts) From 155cf152c91b6aa38e3897287a707fb604ec383c Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 08:25:55 -0500 Subject: [PATCH 13/22] All hosts --- jarvis_util/util/hostfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jarvis_util/util/hostfile.py b/jarvis_util/util/hostfile.py index 373c225..e8c1d1d 100644 --- a/jarvis_util/util/hostfile.py +++ b/jarvis_util/util/hostfile.py @@ -54,7 +54,7 @@ def __init__(self, hostfile=None, all_hosts=None, all_hosts_ip=None, else: self._set_hosts(['localhost']) - self.hosts_iter = [Hostfile(hosts=host) for host in self.hosts] + self.hosts_iter = [Hostfile(all_hosts=host) for host in self.hosts] def _load_hostfile(self, path): """ From 05c231867c5826d1ba4523b3faae65ea6dd039ac Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 08:27:49 -0500 Subject: [PATCH 14/22] Pass host as a list --- jarvis_util/util/hostfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jarvis_util/util/hostfile.py b/jarvis_util/util/hostfile.py index e8c1d1d..588cd5f 100644 --- a/jarvis_util/util/hostfile.py +++ b/jarvis_util/util/hostfile.py @@ -54,7 +54,7 @@ def __init__(self, hostfile=None, all_hosts=None, all_hosts_ip=None, else: self._set_hosts(['localhost']) - self.hosts_iter = [Hostfile(all_hosts=host) for host in self.hosts] + self.hosts_iter = [Hostfile(all_hosts=[host]) for host in self.hosts] def _load_hostfile(self, path): """ From 71e9cb6f39a353a8c82fd0ea0acf41b059ba4d69 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 08:28:15 -0500 Subject: [PATCH 15/22] Find ips --- jarvis_util/util/hostfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jarvis_util/util/hostfile.py b/jarvis_util/util/hostfile.py index 588cd5f..0d3dc93 100644 --- a/jarvis_util/util/hostfile.py +++ b/jarvis_util/util/hostfile.py @@ -54,7 +54,8 @@ def __init__(self, hostfile=None, all_hosts=None, all_hosts_ip=None, else: self._set_hosts(['localhost']) - self.hosts_iter = [Hostfile(all_hosts=[host]) for host in self.hosts] + self.hosts_iter = [Hostfile(all_hosts=[host], find_ips=find_ips) + for host in self.hosts] def _load_hostfile(self, path): """ From 026f309c9614aab9df9ea6f49f7f5d530b5c294b Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 08:29:33 -0500 Subject: [PATCH 16/22] Make list just calaculate the list --- jarvis_util/util/hostfile.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/jarvis_util/util/hostfile.py b/jarvis_util/util/hostfile.py index 0d3dc93..5d015de 100644 --- a/jarvis_util/util/hostfile.py +++ b/jarvis_util/util/hostfile.py @@ -28,7 +28,6 @@ def __init__(self, hostfile=None, all_hosts=None, all_hosts_ip=None, self.hosts = [] self.all_hosts = [] self.all_hosts_ip = [] - self.subhosts = [] self.path = hostfile self.find_ips = find_ips @@ -54,9 +53,6 @@ def __init__(self, hostfile=None, all_hosts=None, all_hosts_ip=None, else: self._set_hosts(['localhost']) - self.hosts_iter = [Hostfile(all_hosts=[host], find_ips=find_ips) - for host in self.hosts] - def _load_hostfile(self, path): """ Expand a hostfile @@ -198,7 +194,7 @@ def save(self, path): return self def list(self): - return self.hosts_iter + return [Hostfile(all_hosts=[host]) for host in self.hosts] def enumerate(self): return enumerate(self.list()) From 0fbb18a7d524f4df3ef0f4bb3535915cbe3ff3a3 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 12:24:46 -0500 Subject: [PATCH 17/22] Begin correcting sudo commands on SSH --- jarvis_util/shell/exec_info.py | 20 +++++++++++++------- jarvis_util/shell/local_exec.py | 9 +++++---- jarvis_util/shell/pssh_exec.py | 1 - jarvis_util/shell/ssh_exec.py | 5 +++-- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/jarvis_util/shell/exec_info.py b/jarvis_util/shell/exec_info.py index cf7b2a2..db32ba1 100644 --- a/jarvis_util/shell/exec_info.py +++ b/jarvis_util/shell/exec_info.py @@ -166,20 +166,26 @@ def set_exit_code(self): def wait(self): pass - def smash_cmd(self, cmds): + def smash_cmd(self, cmds, sudo, basic_env): """ Convert a list of commands into a single command for the shell to execute. :param cmds: A list of commands or a single command string + :param prefix: A prefix for each command :return: """ - if isinstance(cmds, list): - return ' && '.join(cmds) - elif isinstance(cmds, str): - return cmds - else: - raise Exception('Command must be either list or string') + env = None + if sudo: + env = [f'-E {key}=\"{val}\"' for key, val in + basic_env.items()] + env = ' '.join(env) + env = f'sudo {env}' + if not isinstance(cmds, (list, tuple)): + cmds = [cmds] + if env is not None: + cmds = [f'{env} {cmd}' for cmd in cmds] + return ' && '.join(cmds) def wait_list(self, nodes): for node in nodes: diff --git a/jarvis_util/shell/local_exec.py b/jarvis_util/shell/local_exec.py index 8f02785..556eaed 100644 --- a/jarvis_util/shell/local_exec.py +++ b/jarvis_util/shell/local_exec.py @@ -27,7 +27,6 @@ def __init__(self, cmd, exec_info): """ super().__init__() - cmd = self.smash_cmd(cmd) # Managing console output and collection self.collect_output = exec_info.collect_output @@ -56,13 +55,13 @@ def __init__(self, cmd, exec_info): self.exit_code = 0 # Copy ENV + self.basic_env = exec_info.basic_env.copy() self.env = exec_info.env.copy() for key, val in os.environ.items(): if key not in self.env: self.env[key] = val # Managing command execution - self.cmd = cmd self.sudo = exec_info.sudo self.stdin = exec_info.stdin self.exec_async = exec_info.exec_async @@ -71,13 +70,15 @@ def __init__(self, cmd, exec_info): self.cwd = os.getcwd() else: self.cwd = exec_info.cwd + + # Create the command + cmd = self.smash_cmd(cmd, self.sudo, self.basic_env) + self.cmd = cmd if self.jutil.debug_local_exec: print(cmd) self._start_bash_processes() def _start_bash_processes(self): - if self.sudo: - self.cmd = f'sudo {self.cmd}' time.sleep(self.sleep_ms) # pylint: disable=R1732 self.proc = subprocess.Popen(self.cmd, diff --git a/jarvis_util/shell/pssh_exec.py b/jarvis_util/shell/pssh_exec.py index f2c97ca..7aa53b8 100644 --- a/jarvis_util/shell/pssh_exec.py +++ b/jarvis_util/shell/pssh_exec.py @@ -22,7 +22,6 @@ def __init__(self, cmd, exec_info): :param exec_info: Info needed to execute command with SSH """ super().__init__() - self.cmd = self.smash_cmd(cmd) self.exec_async = exec_info.exec_async self.hosts = exec_info.hostfile.hosts self.execs_ = [] diff --git a/jarvis_util/shell/ssh_exec.py b/jarvis_util/shell/ssh_exec.py index 3b95447..ddd421c 100644 --- a/jarvis_util/shell/ssh_exec.py +++ b/jarvis_util/shell/ssh_exec.py @@ -19,13 +19,14 @@ def __init__(self, cmd, exec_info): :param exec_info: Info needed to execute command with SSH """ - cmd = self.smash_cmd(cmd) self.addr = exec_info.hostfile.hosts[0] self.user = exec_info.user self.pkey = exec_info.pkey self.port = exec_info.port self.sudo = exec_info.sudo self.ssh_env = exec_info.env + self.basic_env = exec_info.env + cmd = self.smash_cmd(cmd, self.sudo, self.basic_env) if not exec_info.hostfile.is_local(): super().__init__(self.ssh_cmd(cmd), exec_info.mod(env=exec_info.basic_env)) @@ -50,7 +51,7 @@ def ssh_cmd(self, cmd): cmd_lines.append(f'{key}=\"{val}\"') cmd_lines.append(cmd) env_cmd = ' '.join(cmd_lines) - real_cmd = f'{ssh_cmd} \"{env_cmd}\"' + real_cmd = f'{ssh_cmd} {env_cmd}' return real_cmd From de2b88f46b7d308c8cf91c00cdc157daf61b852f Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 12:32:44 -0500 Subject: [PATCH 18/22] Fix documentation --- jarvis_util/shell/exec_info.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jarvis_util/shell/exec_info.py b/jarvis_util/shell/exec_info.py index db32ba1..673e806 100644 --- a/jarvis_util/shell/exec_info.py +++ b/jarvis_util/shell/exec_info.py @@ -173,6 +173,8 @@ def smash_cmd(self, cmds, sudo, basic_env): :param cmds: A list of commands or a single command string :param prefix: A prefix for each command + :param sudo: Whether or not root is required + :param basic_env: The environment to forward to the command :return: """ env = None From 4eddb33aff33e3c92a950ad488354151e5ed2cb3 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 13:05:53 -0500 Subject: [PATCH 19/22] Specify whether sudo env forwarding is supported --- jarvis_util/shell/exec_info.py | 9 ++++++--- jarvis_util/shell/local_exec.py | 2 +- jarvis_util/shell/ssh_exec.py | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/jarvis_util/shell/exec_info.py b/jarvis_util/shell/exec_info.py index 673e806..31e26a0 100644 --- a/jarvis_util/shell/exec_info.py +++ b/jarvis_util/shell/exec_info.py @@ -34,7 +34,7 @@ class ExecInfo: def __init__(self, exec_type=ExecType.LOCAL, nprocs=None, ppn=None, user=None, pkey=None, port=None, hostfile=None, hosts=None, env=None, - sleep_ms=0, sudo=False, cwd=None, + sleep_ms=0, sudo=False, sudoenv=True, cwd=None, collect_output=None, pipe_stdout=None, pipe_stderr=None, hide_output=None, exec_async=False, stdin=None): """ @@ -50,6 +50,7 @@ def __init__(self, exec_type=ExecType.LOCAL, nprocs=None, ppn=None, :param env: The environment variables to use for command. :param sleep_ms: Sleep for a period of time AFTER executing :param sudo: Execute command with root privilege. E.g., SSH, PSSH + :param sudoenv: Support environment preservation in sudo :param cwd: Set current working directory. E.g., SSH, PSSH :param collect_output: Collect program output in python buffer :param pipe_stdout: Pipe STDOUT into a file. (path string) @@ -72,6 +73,7 @@ def __init__(self, exec_type=ExecType.LOCAL, nprocs=None, ppn=None, self._set_env(env) self.cwd = cwd self.sudo = sudo + self.sudoenv = self.sudoenv self.sleep_ms = sleep_ms self.collect_output = collect_output self.pipe_stdout = pipe_stdout @@ -166,7 +168,7 @@ def set_exit_code(self): def wait(self): pass - def smash_cmd(self, cmds, sudo, basic_env): + def smash_cmd(self, cmds, sudo, basic_env, sudoenv): """ Convert a list of commands into a single command for the shell to execute. @@ -175,10 +177,11 @@ def smash_cmd(self, cmds, sudo, basic_env): :param prefix: A prefix for each command :param sudo: Whether or not root is required :param basic_env: The environment to forward to the command + :param sudoenv: Whether sudo supports environment forwarding :return: """ env = None - if sudo: + if sudo and sudoenv: env = [f'-E {key}=\"{val}\"' for key, val in basic_env.items()] env = ' '.join(env) diff --git a/jarvis_util/shell/local_exec.py b/jarvis_util/shell/local_exec.py index 556eaed..cade8d5 100644 --- a/jarvis_util/shell/local_exec.py +++ b/jarvis_util/shell/local_exec.py @@ -72,7 +72,7 @@ def __init__(self, cmd, exec_info): self.cwd = exec_info.cwd # Create the command - cmd = self.smash_cmd(cmd, self.sudo, self.basic_env) + cmd = self.smash_cmd(cmd, self.sudo, self.basic_env, exec_info.sudoenv) self.cmd = cmd if self.jutil.debug_local_exec: print(cmd) diff --git a/jarvis_util/shell/ssh_exec.py b/jarvis_util/shell/ssh_exec.py index ddd421c..e207971 100644 --- a/jarvis_util/shell/ssh_exec.py +++ b/jarvis_util/shell/ssh_exec.py @@ -26,7 +26,7 @@ def __init__(self, cmd, exec_info): self.sudo = exec_info.sudo self.ssh_env = exec_info.env self.basic_env = exec_info.env - cmd = self.smash_cmd(cmd, self.sudo, self.basic_env) + cmd = self.smash_cmd(cmd, self.sudo, self.basic_env, exec_info.sudoenv) if not exec_info.hostfile.is_local(): super().__init__(self.ssh_cmd(cmd), exec_info.mod(env=exec_info.basic_env)) From 3e7b2c4c8737c645febb4759acda38cda0ff0a6d Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 13:09:40 -0500 Subject: [PATCH 20/22] Specify whether sudo env forwarding is supported --- jarvis_util/shell/exec_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jarvis_util/shell/exec_info.py b/jarvis_util/shell/exec_info.py index 31e26a0..10c3bd9 100644 --- a/jarvis_util/shell/exec_info.py +++ b/jarvis_util/shell/exec_info.py @@ -73,7 +73,7 @@ def __init__(self, exec_type=ExecType.LOCAL, nprocs=None, ppn=None, self._set_env(env) self.cwd = cwd self.sudo = sudo - self.sudoenv = self.sudoenv + self.sudoenv = sudoenv self.sleep_ms = sleep_ms self.collect_output = collect_output self.pipe_stdout = pipe_stdout From 7bae94810cf1048787e9e34512eaa54082c33030 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 19 Sep 2023 13:15:20 -0500 Subject: [PATCH 21/22] Specify whether sudo env forwarding is supported --- jarvis_util/shell/exec_info.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/jarvis_util/shell/exec_info.py b/jarvis_util/shell/exec_info.py index 10c3bd9..94108fd 100644 --- a/jarvis_util/shell/exec_info.py +++ b/jarvis_util/shell/exec_info.py @@ -181,10 +181,12 @@ def smash_cmd(self, cmds, sudo, basic_env, sudoenv): :return: """ env = None - if sudo and sudoenv: - env = [f'-E {key}=\"{val}\"' for key, val in - basic_env.items()] - env = ' '.join(env) + if sudo: + env = '' + if sudoenv: + env = [f'-E {key}=\"{val}\"' for key, val in + basic_env.items()] + env = ' '.join(env) env = f'sudo {env}' if not isinstance(cmds, (list, tuple)): cmds = [cmds] From 18bb668dabad78188c4e9df67346df8ae6bd7629 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Thu, 21 Sep 2023 16:41:21 -0500 Subject: [PATCH 22/22] Add a real_kwargs parameter, without default values filled in --- jarvis_util/util/argparse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jarvis_util/util/argparse.py b/jarvis_util/util/argparse.py index daa6566..2cc7dac 100644 --- a/jarvis_util/util/argparse.py +++ b/jarvis_util/util/argparse.py @@ -45,6 +45,7 @@ def __init__(self, args=None, exit_on_fail=True, **custom_info): self.menu = None self.menu_name = None self.kwargs = {} + self.real_kwargs = {} self.define_options() self._parse() @@ -190,6 +191,7 @@ def _parse(self): default_args = self.default_kwargs( list(self.menu['kw_opts'].values()) + self.menu['pos_opts']) default_args.update(self.kwargs) + self.real_kwargs = self.kwargs self.kwargs = default_args @staticmethod