From 57852580b61c091c739e9cb7d72548fb44cbddd2 Mon Sep 17 00:00:00 2001 From: Bo Peng Date: Fri, 16 Feb 2024 00:50:47 -0600 Subject: [PATCH] Fix tests --- src/sos/functions.py | 1 + src/sos/hosts.py | 13 +-- src/sos/parser.py | 3 +- src/sos/syntax.py | 2 - src/sos/templates/parts/hover_doc.tpl | 4 +- test/build_test_docker.sh | 5 +- test/test_remote.py | 119 +------------------------- 7 files changed, 11 insertions(+), 136 deletions(-) diff --git a/src/sos/functions.py b/src/sos/functions.py index 3637525d8..6c227b5d5 100644 --- a/src/sos/functions.py +++ b/src/sos/functions.py @@ -33,6 +33,7 @@ def _load_actions(): def sos_run_script(action, script, *args, **kwargs): + '''Call script-execution actions.''' if not g_action_map: _load_actions() try: diff --git a/src/sos/hosts.py b/src/sos/hosts.py index 028de1e05..22ce510b3 100755 --- a/src/sos/hosts.py +++ b/src/sos/hosts.py @@ -496,7 +496,7 @@ def _prepare_task(self, task_id): # map variables runtime["_runtime"]["workdir"] = ( - task_vars["_runtime"]["workdir"] if "workdir" in task_vars["_runtime"] else path.cwd().to_named_path()) + task_vars["_runtime"]["workdir"] if "workdir" in task_vars["_runtime"] else str(path.cwd())) if runtime["_runtime"]["workdir"].startswith("#"): try: @@ -679,18 +679,9 @@ def receive_result(self, task_id: str) -> Dict[str, int]: job_dict = params.sos_dict if ("_output" in job_dict and job_dict["_output"] and not isinstance(job_dict["_output"], Undetermined) and env.config["run_mode"] != "dryrun"): - received = self.receive_from_host([x for x in job_dict["_output"] if isinstance(x, (str, path))]) + received = {x:x for x in job_dict["_output"] if isinstance(x, (str, path))} if received: env.logger.info(f"{task_id} ``received`` {short_repr(received.keys())} from {self.alias}") - if "from_host" in job_dict["_runtime"] and env.config["run_mode"] != "dryrun": - if isinstance(job_dict["_runtime"]["from_host"], (Sequence, str)): - received = self.receive_from_host(job_dict["_runtime"]["from_host"]) - if received: - env.logger.info(f"{task_id} ``received`` {short_repr(received.keys())} from {self.alias}") - else: - env.logger.warning( - f"Expecting a string or list of string from from_host: {job_dict['_runtime']['from_host']} received" - ) # we need to translate result from remote path to local if "output" in res: if "_output" not in job_dict: diff --git a/src/sos/parser.py b/src/sos/parser.py index 7875b2a9e..87b8f4dd3 100755 --- a/src/sos/parser.py +++ b/src/sos/parser.py @@ -490,7 +490,8 @@ def wrap_script(self) -> None: self._script = repr(self._script) self.statements[-1] = [ "!", - f'sos_run_script("{self._action}", {self._script}{(", " + opt) if opt else ""})\n', + #f'sos_run_script("{self._action}", {self._script}{(", " + opt) if opt else ""})\n', + f'{self._action}({self._script}{(", " + opt) if opt else ""})\n', ] self.values = [] self._action = None diff --git a/src/sos/syntax.py b/src/sos/syntax.py index 05c263689..fb932cdfe 100644 --- a/src/sos/syntax.py +++ b/src/sos/syntax.py @@ -28,8 +28,6 @@ "env", "prepend_path", "queue", - "to_host", - "from_host", "trunk_size", "trunk_workers", "tags", diff --git a/src/sos/templates/parts/hover_doc.tpl b/src/sos/templates/parts/hover_doc.tpl index 091d97138..8e7ab68e7 100644 --- a/src/sos/templates/parts/hover_doc.tpl +++ b/src/sos/templates/parts/hover_doc.tpl @@ -26,8 +26,8 @@ let sos_keywords = new Set([ 'sos_run', 'fail_if', 'done_if', 'warn_if', 'skip_if', 'get_output', 'expand_pattern', // task options - 'walltime', 'cores', 'mem', 'queue', 'to_host', 'nodes', - 'from_host', 'trunk_size', 'trunk_workers', 'workdir', + 'walltime', 'cores', 'mem', 'queue', 'nodes', + 'trunk_size', 'trunk_workers', 'workdir', 'shared', 'env', 'prepend_path', 'tags', // targets 'file_target', 'executable', 'sos_variable', 'env_variable', diff --git a/test/build_test_docker.sh b/test/build_test_docker.sh index f4452c6e6..9282a140f 100644 --- a/test/build_test_docker.sh +++ b/test/build_test_docker.sh @@ -42,8 +42,7 @@ ADD authorized_keys /root/.ssh/authorized_keys ARG SHA=LATEST RUN SHA=$SHA git clone http://github.com/vatlab/sos sos -RUN cd sos && pip install . -U -RUN pip install sos-pbs +RUN pip install sos sos-pbs RUN echo "export TS_SLOTS=10" >> /root/.bash_profile @@ -62,7 +61,7 @@ docker build --build-arg SHA=$SHA -t eg_sshd . # # start docker image -docker run -d -P --env TS_SLOTS=10 --name test_sos eg_sshd +docker run -d -v `pwd`:`pwd` -P --env TS_SLOTS=10 --name test_sos eg_sshd # get the port PORT22=$(docker port test_sos 22 | cut -f2 -d:) diff --git a/test/test_remote.py b/test/test_remote.py index 076dd8ff9..f0b8ef79c 100644 --- a/test/test_remote.py +++ b/test/test_remote.py @@ -23,63 +23,6 @@ has_docker = False - -@pytest.mark.skipif(not has_docker, reason="Docker container not usable") -def test_to_host_option(clear_now_and_after): - """Test from_remote option""" - clear_now_and_after("to_host_testfile.txt", "to_host_linecount.txt") - with open("to_host_testfile.txt", "w") as tf: - for i in range(100): - tf.write(f"line {i+1}\n") - execute_workflow( - """ - [10] - output: 'to_host_linecount.txt' - task: to_host='to_host_testfile.txt' - sh: - wc -l to_host_testfile.txt > to_host_linecount.txt - """, - options={ - "config_file": "~/docker.yml", - "default_queue": "docker", - "sig_mode": "force", - }, - ) - assert os.path.isfile("to_host_linecount.txt") - with open("to_host_linecount.txt") as lc: - assert lc.read().strip().startswith("100") - - - -@pytest.mark.skipif(not has_docker, reason="Docker container not usable") -def test_to_host_option_with_named_path(clear_now_and_after): - """Test from_remote option""" - clear_now_and_after( - os.path.expanduser("~/to_host_named_testfile.txt"), - "to_host_named_linecount.txt", - ) - with open(os.path.expanduser("~/to_host_named_testfile.txt"), "w") as tf: - for i in range(200): - tf.write(f"line {i+1}\n") - execute_workflow( - """ - [10] - output: 'to_host_named_linecount.txt' - task: to_host='#home/to_host_named_testfile.txt' - sh: - wc -l ~/to_host_named_testfile.txt > to_host_named_linecount.txt - """, - options={ - "config_file": "~/docker.yml", - "default_queue": "docker", - "sig_mode": "force", - }, - ) - assert os.path.isfile("to_host_named_linecount.txt") - with open("to_host_named_linecount.txt") as lc: - assert lc.read().strip().startswith("200") - - def test_worker_procs(): # test -j option execute_workflow( @@ -117,7 +60,6 @@ def test_worker_procs_with_task(): ) - @pytest.mark.skipif(not has_docker, reason="Docker container not usable") def test_remote_execute(clear_now_and_after, script_factory): clear_now_and_after("result_remote.txt", "result_remote1.txt", "local.txt") @@ -180,11 +122,9 @@ def test_remote_workflow_remote_queue(script_factory): assert 0 == subprocess.call(f"sos run {test_r_q} -c ~/docker.yml -r ts -q ts", shell=True) - @pytest.mark.skipif(not has_docker, reason="Docker container not usable") def test_remote_exec(clear_now_and_after): clear_now_and_after("result_exec.txt") - root_dir = "/root/build" if "TRAVIS" in os.environ else "/root" execute_workflow( """ output: 'result_exec.txt' @@ -204,66 +144,11 @@ def test_remote_exec(clear_now_and_after): with open(file_target("result_exec.txt")) as res: result = res.read() assert "Output: result_exec.txt" in result - assert f"PWD: {root_dir}" in result - - - -@pytest.mark.skipif(not has_docker, reason="Docker container not usable") -def test_remote_exec_named_path(clear_now_and_after): - clear_now_and_after("result_named_path.txt") - root_dir = "/root/build" if "TRAVIS" in os.environ else "/root" - - execute_workflow( - """ - output: '#home/result_named_path.txt' - - task: - sh: expand=True - echo Output: {_output} > {_output} - echo PWD: `pwd`. >> {_output} - """, - options={ - "config_file": "~/docker.yml", - "default_queue": "docker", - "sig_mode": "force", - }, - ) - assert file_target("#home/result_named_path.txt").target_exists() - with open(file_target("#home/result_named_path.txt")) as res: - result = res.read() - print(result) - assert "Output: /root/result_named_path.txt" in result - assert f"PWD: {root_dir}" in result - - - -@pytest.mark.skipif(not has_docker, reason="Docker container not usable") -def test_remote_exec_workdir_named_path(clear_now_and_after): - clear_now_and_after(file_target("#home/wd/result_workdir_named_path.txt")) - execute_workflow( - """ - output: '#home/wd/result_workdir_named_path.txt' - - task: workdir='/root' - sh: expand=True - echo Output: {_output} > {_output} - echo PWD: `pwd`. >> {_output} - """, - options={ - "config_file": "~/docker.yml", - "default_queue": "docker", - "sig_mode": "force", - }, - ) - assert file_target("#home/wd/result_workdir_named_path.txt").target_exists() - with open(file_target("#home/wd/result_workdir_named_path.txt")) as res: - result = res.read() - assert "Output: /root/wd/result_workdir_named_path.txt" in result - assert "PWD: /root." in result + assert f"PWD: {os.getcwd()}" in result @pytest.mark.skipif(not has_docker, reason="Docker container not usable") -def test_remote_exec_workdir_wo_named_path(clear_now_and_after): +def test_remote_exec_workdir(clear_now_and_after): clear_now_and_after(file_target("result_workdir_wo_named.txt")) with pytest.raises(Exception): execute_workflow(