From d148337f65cbd754b95c157a310568e46775a2e2 Mon Sep 17 00:00:00 2001 From: Bo Peng Date: Mon, 12 Feb 2024 17:16:27 -0600 Subject: [PATCH] Code cleanup --- src/sos/__main__.py | 2 +- src/sos/actions.py | 3 +- src/sos/converter.py | 2 +- src/sos/dag.py | 6 ++-- src/sos/eval.py | 6 ++-- src/sos/executor_utils.py | 2 +- src/sos/hosts.py | 6 ++-- src/sos/parser.py | 4 +-- src/sos/pattern.py | 2 +- src/sos/preview.py | 4 +-- src/sos/remote.py | 2 +- src/sos/signatures.py | 1 - src/sos/step_executor.py | 22 +++++++++------ src/sos/syntax.py | 2 +- src/sos/targets.py | 4 +-- src/sos/targets_python.py | 4 +-- src/sos/targets_r.py | 2 +- src/sos/task_executor.py | 22 +++++++++------ src/sos/tasks.py | 11 ++++---- src/sos/utils.py | 16 +++++------ src/sos/workers.py | 5 ++-- src/sos/workflow_engines.py | 2 +- src/sos/workflow_executor.py | 15 ++++++---- src/sos/workflow_report.py | 2 +- test/test_parser.py | 16 +++++------ test/test_utils.py | 54 ++++++++++++------------------------ 26 files changed, 105 insertions(+), 112 deletions(-) diff --git a/src/sos/__main__.py b/src/sos/__main__.py index e634f0584..efefa41a5 100755 --- a/src/sos/__main__.py +++ b/src/sos/__main__.py @@ -2203,7 +2203,7 @@ def cmd_remove(args, unknown_args): # a special case where all file and runtime signatures are removed. # no other options are allowed. if sig_files: - sig_ids = list(set([x[0] for x in sig_files])) + sig_ids = list({x[0] for x in sig_files}) step_signatures = StepSignatures() num_removed_steps = step_signatures.remove_many(sig_ids) if not num_removed_steps: diff --git a/src/sos/actions.py b/src/sos/actions.py index 8986184a7..f3c0c0de4 100644 --- a/src/sos/actions.py +++ b/src/sos/actions.py @@ -33,7 +33,8 @@ from .parser import SoS_Script from .syntax import SOS_ACTION_OPTIONS from .targets import executable, file_target, path, paths, sos_targets -from .utils import (StopInputGroup, TerminateExecution, TimeoutInterProcessLock, env, fileMD5, get_traceback, +from .utils import (StopInputGroup, TerminateExecution, + TimeoutInterProcessLock, env, fileMD5, get_traceback, load_config_files, short_repr, textMD5, transcribe) __all__ = [ diff --git a/src/sos/converter.py b/src/sos/converter.py index bf913a52d..92872ea08 100755 --- a/src/sos/converter.py +++ b/src/sos/converter.py @@ -116,7 +116,7 @@ def analyse_text(self, text): ] -class ScriptToHTMLConverter(object): +class ScriptToHTMLConverter: def __init__(self, *args, **kwargs): pass diff --git a/src/sos/dag.py b/src/sos/dag.py index 7407066e3..cf3af3e7c 100644 --- a/src/sos/dag.py +++ b/src/sos/dag.py @@ -76,7 +76,7 @@ # -class SoS_Node(object): +class SoS_Node: def __init__(self, step_uuid: str, node_name: str, wf_index: Union[int, None], node_index: Union[int, None], @@ -277,10 +277,10 @@ def dangling(self, targets: sos_targets): else: missing.add(x) else: - missing = set([ + missing = { x for x in self._all_depends_files.keys() if x not in self._all_output_files and not x.target_exists() - ]) + } for x in targets: if x not in self._all_output_files: if x.target_exists('target'): diff --git a/src/sos/eval.py b/src/sos/eval.py index 9924d6c69..6f96713d8 100644 --- a/src/sos/eval.py +++ b/src/sos/eval.py @@ -273,7 +273,7 @@ def _is_expr(expr): return False -class StatementHash(object): +class StatementHash: stmt_hash = {} def __init__(self) -> None: @@ -391,7 +391,7 @@ def SoS_exec(script: str, # -class Undetermined(object): +class Undetermined: def __init__(self, expr: str = "") -> None: if not isinstance(expr, str): @@ -414,7 +414,7 @@ def targets(self) -> "Undetermined": return self -class on_demand_options(object): +class on_demand_options: """Expression that will be evaluated upon request.""" def __init__(self, items: Optional[Dict[str, Any]]) -> None: diff --git a/src/sos/executor_utils.py b/src/sos/executor_utils.py index 83595902e..ba8f4efac 100644 --- a/src/sos/executor_utils.py +++ b/src/sos/executor_utils.py @@ -12,9 +12,9 @@ import traceback from collections.abc import Sequence from io import StringIO +from secrets import token_hex from tokenize import generate_tokens from typing import Any -from secrets import token_hex import psutil diff --git a/src/sos/hosts.py b/src/sos/hosts.py index 85c41984f..ccf89f7dc 100755 --- a/src/sos/hosts.py +++ b/src/sos/hosts.py @@ -92,7 +92,7 @@ def run(self): sys.stdout.flush() sys.stderr.flush() try: - si = open(os.devnull, "r") + si = open(os.devnull) so = open(os.devnull, "w") se = open(os.devnull, "w") os.dup2(si.fileno(), sys.stdin.fileno()) @@ -127,7 +127,7 @@ def _show_err_and_out(task_id, res) -> None: sys.stderr.write("\n") -class LocalHost(object): +class LocalHost: """For local host, no path map, send and receive ...""" def __init__( @@ -297,7 +297,7 @@ def receive_result(self, task_id: str) -> Dict[str, Any]: return res -class RemoteHost(object): +class RemoteHost: """A remote host class that manages how to communicate with remote host""" def __init__( diff --git a/src/sos/parser.py b/src/sos/parser.py index b22fae1c4..f4706a271 100755 --- a/src/sos/parser.py +++ b/src/sos/parser.py @@ -87,13 +87,13 @@ def is_type_hint(stmt: str) -> bool: # input: variable # if "=" not in stmt: - action, par = [x.strip() for x in stmt.split(":", 1)] + action, par = (x.strip() for x in stmt.split(":", 1)) else: # one parameter? # # action: input={'a': b} # - action, par = [x.strip() for x in stmt.split("=", 1)[0].split(":", 1)] + action, par = (x.strip() for x in stmt.split("=", 1)[0].split(":", 1)) if action in SOS_DIRECTIVES: return False diff --git a/src/sos/pattern.py b/src/sos/pattern.py index c57197cc2..3f1c6dd4c 100644 --- a/src/sos/pattern.py +++ b/src/sos/pattern.py @@ -141,7 +141,7 @@ def expand_pattern(pattern: str) -> List[str]: if key not in env.sos_dict: raise ValueError(f"Undefined variable {key} in pattern {pattern}") if not isinstance(env.sos_dict[key], str) and isinstance( - env.sos_dict[key], collections.Sequence): + env.sos_dict[key], collections.abc.Sequence): if sz is None: sz = len(env.sos_dict[key]) wildcard = [copy.deepcopy(wildcard[0]) for x in range(sz)] diff --git a/src/sos/preview.py b/src/sos/preview.py index 388825905..616442b50 100644 --- a/src/sos/preview.py +++ b/src/sos/preview.py @@ -104,7 +104,7 @@ def preview_img(filename, kernel=None, style=None): def preview_svg(filename, kernel=None, style=None): - with open(filename, "r") as f: + with open(filename) as f: image_data = f.read() return {"image/svg+xml": image_data} @@ -255,7 +255,7 @@ def preview_txt(filename, kernel=None, style=None): hint = f' ({limit} displayed, see --limit)' if nlines > limit else '' content = f"HINT: {nlines} line{'s' if nlines > 1 else ''}{hint}\n" - with open(filename, "r") as fin: + with open(filename) as fin: if limit < 0: content += fin.read() else: diff --git a/src/sos/remote.py b/src/sos/remote.py index 9bca8b0cf..c09a0c84b 100755 --- a/src/sos/remote.py +++ b/src/sos/remote.py @@ -248,7 +248,7 @@ def test_paths(host): f"Failed to receive file from remote host {remote}: file does not exist" ) # check file content? - with open(os.path.join(local, f".sos_test_{tID}.txt"), "r") as tFile: + with open(os.path.join(local, f".sos_test_{tID}.txt")) as tFile: remote_content = tFile.read() if remote_content != str(tID): return f"Content of received file does not match: {tID} expected, {remote_content} received." diff --git a/src/sos/signatures.py b/src/sos/signatures.py index 97ff92910..493634676 100644 --- a/src/sos/signatures.py +++ b/src/sos/signatures.py @@ -213,4 +213,3 @@ def clear(self): except sqlite3.DatabaseError as e: env.logger.warning(f"Failed to clear workflow database: {e}") return [] - \ No newline at end of file diff --git a/src/sos/step_executor.py b/src/sos/step_executor.py index 6dec4584f..5b5356ce9 100755 --- a/src/sos/step_executor.py +++ b/src/sos/step_executor.py @@ -17,16 +17,20 @@ from .controller import close_socket, create_socket, send_message_to_controller from .eval import KeepOnlyImportAndDefine, SoS_eval, SoS_exec, accessed_vars -from .executor_utils import (ExecuteError, __named_output__, __null_func__, __output_from__, __traced__, clear_output, - create_task, get_traceback_msg, reevaluate_output, statementMD5, validate_step_sig, - verify_input) +from .executor_utils import (ExecuteError, __named_output__, __null_func__, + __output_from__, __traced__, clear_output, + create_task, get_traceback_msg, reevaluate_output, + statementMD5, validate_step_sig, verify_input) from .messages import decode_msg, encode_msg -from .syntax import (SOS_DEPENDS_OPTIONS, SOS_INPUT_OPTIONS, SOS_OUTPUT_OPTIONS, SOS_TARGETS_OPTIONS) -from .targets import (RemovedTarget, RuntimeInfo, UnavailableLock, UnknownTarget, dynamic, file_target, invalid_target, +from .syntax import (SOS_DEPENDS_OPTIONS, SOS_INPUT_OPTIONS, + SOS_OUTPUT_OPTIONS, SOS_TARGETS_OPTIONS) +from .targets import (RemovedTarget, RuntimeInfo, UnavailableLock, + UnknownTarget, dynamic, file_target, invalid_target, sos_step, sos_targets, sos_variable) from .tasks import MasterTaskParams, TaskFile -from .utils import (ArgumentError, ProcessKilled, StopInputGroup, TerminateExecution, env, get_localhost_ip, - get_traceback, short_repr, textMD5) +from .utils import (ArgumentError, ProcessKilled, StopInputGroup, + TerminateExecution, env, get_localhost_ip, get_traceback, + short_repr, textMD5) __all__: List = [] @@ -510,8 +514,8 @@ def process_output_args(self, ofiles: sos_targets, **kwargs): # create directory if ofiles.valid(): - parents = set( - [os.path.abspath(os.path.join(ofile, os.pardir)) for ofile in ofiles if isinstance(ofile, file_target)]) + parents = { + os.path.abspath(os.path.join(ofile, os.pardir)) for ofile in ofiles if isinstance(ofile, file_target)} for parent_dir in parents: if parent_dir and not os.path.isdir(parent_dir): os.makedirs(parent_dir, exist_ok=True) diff --git a/src/sos/syntax.py b/src/sos/syntax.py index 3aa5f00a2..05c263689 100644 --- a/src/sos/syntax.py +++ b/src/sos/syntax.py @@ -99,7 +99,7 @@ # -class LazyRegex(object): +class LazyRegex: """A proxy around a real regex, which won't be compiled until accessed.""" # These are the parameters on a real _sre.SRE_Pattern object, which we diff --git a/src/sos/targets.py b/src/sos/targets.py index afb253e69..df43b9073 100644 --- a/src/sos/targets.py +++ b/src/sos/targets.py @@ -89,7 +89,7 @@ def __init__(self, signature): # -class BaseTarget(object): +class BaseTarget: """A base class for all targets (e.g. a file)""" def __init__(self, *args, **kwargs): @@ -828,7 +828,7 @@ def validate(self, sig=None): md5_file = self + '.md5' if md5_file.exists(): # validate against md5 - with open(md5_file, 'r') as mfile: + with open(md5_file) as mfile: return mfile.readline().strip().split()[-1] == fileMD5(self, sig_type='full') if sig is not None: sig_mtime, sig_size, sig_md5 = sig diff --git a/src/sos/targets_python.py b/src/sos/targets_python.py index 9564837b8..31a8a2c43 100644 --- a/src/sos/targets_python.py +++ b/src/sos/targets_python.py @@ -24,9 +24,9 @@ def __init__(self, module, version=None, autoinstall=False): raise ValueError( f"Specifying 'version=' option in addition to '{module}' is not allowed" ) - self._module, self._version = [ + self._module, self._version = ( x.strip() for x in self._module.split(opt, 1) - ] + ) if ',' in self._version: raise ValueError( f'SoS does not yet support multiple version comparisons. {self._mdoule} provided' diff --git a/src/sos/targets_r.py b/src/sos/targets_r.py index 1f9c5a39b..6fd25ccea 100644 --- a/src/sos/targets_r.py +++ b/src/sos/targets_r.py @@ -56,7 +56,7 @@ def _install(self, name, version, repos): raise ValueError( f"Specifying 'version=' option in addition to '{name}' is not allowed" ) - name, version = [x.strip() for x in name.split(opt, 1)] + name, version = (x.strip() for x in name.split(opt, 1)) if "," in version: raise ValueError( f"SoS does not yet support multiple version comparisons. {version} provided" diff --git a/src/sos/task_executor.py b/src/sos/task_executor.py index 12714ea57..9ad304293 100644 --- a/src/sos/task_executor.py +++ b/src/sos/task_executor.py @@ -15,23 +15,29 @@ import zmq -from .controller import (Controller, close_socket, connect_controllers, create_socket, disconnect_controllers, - request_answer_from_controller, send_message_to_controller) +from .controller import (Controller, close_socket, connect_controllers, + create_socket, disconnect_controllers, + request_answer_from_controller, + send_message_to_controller) from .eval import SoS_eval, SoS_exec -from .executor_utils import (__null_func__, clear_output, get_traceback_msg, prepare_env) +from .executor_utils import (__null_func__, clear_output, get_traceback_msg, + prepare_env) from .messages import decode_msg from .monitor import TaskMonitor from .step_executor import parse_shared_vars -from .targets import (InMemorySignature, dynamic, file_target, path, sos_step, sos_targets) -from .tasks import (TaskFile, combine_results, monitor_interval, remove_task_files, resource_monitor_interval) -from .utils import (ProcessKilled, StopInputGroup, env, get_localhost_ip, pickleable) +from .targets import (InMemorySignature, dynamic, file_target, path, sos_step, + sos_targets) +from .tasks import (TaskFile, combine_results, monitor_interval, + remove_task_files, resource_monitor_interval) +from .utils import (ProcessKilled, StopInputGroup, env, get_localhost_ip, + pickleable) def signal_handler(*args, **kwargs): raise ProcessKilled() -class BaseTaskExecutor(object): +class BaseTaskExecutor: """Task executor used to execute specified tasks. Any customized task executor should derive from this class. """ @@ -263,7 +269,7 @@ def execute_single_task(self, task_id, params, runtime, sig_content, quiet=False if not os.path.isfile(logfile): raise ValueError(f"logfile {logfile} does not exist after the completion of task") try: - with open(logfile, "r") as log: + with open(logfile) as log: my_stdout.write(f"logfile: {logfile}\n") my_stdout.write(log.read()) except Exception as e: diff --git a/src/sos/tasks.py b/src/sos/tasks.py index 7dfe6204c..80845754e 100644 --- a/src/sos/tasks.py +++ b/src/sos/tasks.py @@ -18,14 +18,15 @@ import fasteners from .targets import sos_targets -from .utils import (DelayedAction, env, expand_size, expand_time, format_duration, format_HHMMSS, linecount_of_file, +from .utils import (DelayedAction, env, expand_size, expand_time, + format_duration, format_HHMMSS, linecount_of_file, pretty_size, sample_lines, short_repr, tail_of_file) monitor_interval = 5 resource_monitor_interval = 60 -class TaskParams(object): +class TaskParams: """A parameter object that encaptulates parameters sending to task executors. This would makes the output of workers, especially in the web interface much cleaner (issue #259)""" @@ -295,7 +296,7 @@ class TaskStatus(Enum): completed = 6 -class TaskFile(object): +class TaskFile: """ The task file has the following format: @@ -526,7 +527,7 @@ def add_result(self, result: dict = {}): params = self._get_params() # this is a master task, get all sub task IDs if hasattr(params, "task_stack"): - missing_tasks = set([x[0] for x in params.task_stack]) + missing_tasks = {x[0] for x in params.task_stack} # cache_file = os.path.join(os.path.expanduser("~"), ".sos", "tasks", self.task_id + ".cache") results = [] @@ -1725,7 +1726,7 @@ def purge_tasks(tasks, purge_all=None, age=None, status=None, tags=None, verbosi all_tasks = [x for x in all_tasks if any(x in tags for x in TaskFile(x[0]).tags.split())] # # remoe all task files - all_tasks = set([x[0] for x in all_tasks]) + all_tasks = {x[0] for x in all_tasks} if all_tasks: # # find all related files, including those in nested directories diff --git a/src/sos/utils.py b/src/sos/utils.py index 615f04dcd..667af518f 100644 --- a/src/sos/utils.py +++ b/src/sos/utils.py @@ -184,7 +184,7 @@ def short_repr(obj, noneAsNA=False): # -class WorkflowDict(object): +class WorkflowDict: """A dictionary object that keeps all SoS workflow objects. IMPORTANT: @@ -384,7 +384,7 @@ def fileMD5(filename, sig_type="partial"): # read overlap_size bytes. elif loc > second_stop: # and < third_stop partial_sig.update(data[-overlap_size:]) - except IOError as e: + except OSError as e: sys.exit(f"Failed to read {filename}: {e}") if full_sig and partial_sig: return partial_sig.hexdigest(), full_sig.hexdigest() @@ -398,7 +398,7 @@ def fileMD5(filename, sig_type="partial"): # -class RuntimeEnvironments(object): +class RuntimeEnvironments: """A singleton object that provides runtime environment for SoS. Atributes of this object include: @@ -417,7 +417,7 @@ class RuntimeEnvironments(object): def __new__(cls, *args, **kwargs): if not cls._instance: - cls._instance = super(RuntimeEnvironments, cls).__new__(cls) + cls._instance = super().__new__(cls) return cls._instance def __init__(self): @@ -512,7 +512,7 @@ def reset(self): "SOS_DEBUG": set(), }) if "SOS_DEBUG" in os.environ: - self.config["SOS_DEBUG"] = set([x for x in os.environ["SOS_DEBUG"].split(",") if "." not in x and x != "-"]) + self.config["SOS_DEBUG"] = {x for x in os.environ["SOS_DEBUG"].split(",") if "." not in x and x != "-"} # # global dictionaries used by SoS during the # execution of SoS workflows @@ -1533,7 +1533,7 @@ def tail_of_file(filename, n, ansi2html=False): while 1: try: f.seek(-(avg_line_length * to_read), 2) - except IOError: + except OSError: # woops. apparently file is smaller than what we want # to step back, go to the beginning instead f.seek(0) @@ -1894,7 +1894,7 @@ def get_nodelist(): env.log_to_file("WORKER", f'Using "-j {args}" on a SLURM cluster.') return args if "PBS_ENVIRONMENT" in os.environ: - with open(os.environ["PBS_NODEFILE"], "r") as hosts: + with open(os.environ["PBS_NODEFILE"]) as hosts: hostlist = hosts.read().split() from collections import Counter, OrderedDict @@ -1912,7 +1912,7 @@ def get_nodelist(): # run on the host, the third entry the name of the queue, # and the fourth entry a processor range to be used in # case of a multiprocessor machine. - with open(os.environ["PE_HOSTFILE"], "r") as hosts: + with open(os.environ["PE_HOSTFILE"]) as hosts: args = [":".join(host.split()[:2]) for host in hosts] env.log_to_file("WORKER", f'Using "-j {args}" on a SGE cluster.') return args diff --git a/src/sos/workers.py b/src/sos/workers.py index 0c9bd98cf..8ce7ab033 100755 --- a/src/sos/workers.py +++ b/src/sos/workers.py @@ -24,7 +24,7 @@ def signal_handler(*args, **kwargs): raise ProcessKilled() -class Runner(object): +class Runner: """ This runner class takea a generator function and run it. 1. When the generator returns None, continue to run without yielding. @@ -451,7 +451,7 @@ def run_task(self, work): env.result_socket.send(encode_msg(res)) -class WorkerManager(object): +class WorkerManager: # manager worker processes def __init__(self, worker_procs, backend_socket): @@ -757,4 +757,3 @@ def kill_all(self): # join all local processes for worker in self._local_workers: worker.join() - \ No newline at end of file diff --git a/src/sos/workflow_engines.py b/src/sos/workflow_engines.py index e2c094544..1f5e027b6 100644 --- a/src/sos/workflow_engines.py +++ b/src/sos/workflow_engines.py @@ -631,7 +631,7 @@ def purge_workflows(workflows, ] # # remoe all workflow files - all_workflows = set([x[0] for x in all_workflows]) + all_workflows = {x[0] for x in all_workflows} if all_workflows: # # find all related files, including those in nested directories diff --git a/src/sos/workflow_executor.py b/src/sos/workflow_executor.py index a7428fef6..fafddece8 100755 --- a/src/sos/workflow_executor.py +++ b/src/sos/workflow_executor.py @@ -18,8 +18,10 @@ import zmq -from .controller import (Controller, close_socket, connect_controllers, create_socket, disconnect_controllers, - request_answer_from_controller, send_message_to_controller) +from .controller import (Controller, close_socket, connect_controllers, + create_socket, disconnect_controllers, + request_answer_from_controller, + send_message_to_controller) from .dag import SoS_DAG from .eval import analyze_global_statements from .executor_utils import ExecuteError, prepare_env @@ -28,8 +30,9 @@ from .parser import SoS_Workflow from .pattern import extract_pattern from .section_analyzer import analyze_section -from .targets import (BaseTarget, RemovedTarget, UnavailableLock, UnknownTarget, file_target, invalid_target, - named_output, path, paths, sos_step, sos_targets, sos_variable) +from .targets import (BaseTarget, RemovedTarget, UnavailableLock, + UnknownTarget, file_target, invalid_target, named_output, + path, paths, sos_step, sos_targets, sos_variable) from .utils import env, get_localhost_ip, pickleable, short_repr, textMD5 from .workflow_report import render_report @@ -53,7 +56,7 @@ def __repr__(self): return self._name -class ProcInfo(object): +class ProcInfo: def __init__(self, socket, port, step) -> None: self.socket = socket @@ -78,7 +81,7 @@ def is_pending(self) -> bool: return self.step._status.endswith("_pending") -class ExecutionManager(object): +class ExecutionManager: """ Execution manager that manages sockets and corresponding steps. For nested workflows (dummy=True), a poller will be created. diff --git a/src/sos/workflow_report.py b/src/sos/workflow_report.py index 08380173e..47ff6e850 100644 --- a/src/sos/workflow_report.py +++ b/src/sos/workflow_report.py @@ -14,7 +14,7 @@ from .utils import dot_to_gif, env, format_duration -class WorkflowSig(object): +class WorkflowSig: def __init__(self, workflow_id): self.data = defaultdict(lambda: defaultdict(list)) diff --git a/test/test_parser.py b/test/test_parser.py index 3d33e40f1..8a1d56f44 100644 --- a/test/test_parser.py +++ b/test/test_parser.py @@ -332,10 +332,10 @@ def test_parameters_section(): """) wf = script.workflow() Base_Executor(wf, args=["--b"]).run(mode="dryrun") - assert env.sos_dict["b"] == True + assert env.sos_dict["b"] is True env.sos_dict.pop("b") Base_Executor(wf, args=["--no-b"]).run(mode="dryrun") - assert env.sos_dict["b"] == False + assert env.sos_dict["b"] is False env.sos_dict.pop("b") # bool with default True script = SoS_Script(""" @@ -344,13 +344,13 @@ def test_parameters_section(): """) wf = script.workflow() Base_Executor(wf, args=[]).run(mode="dryrun") - assert env.sos_dict["b"] == True + assert env.sos_dict["b"] is True env.sos_dict.pop("b") Base_Executor(wf, args=["--b"]).run(mode="dryrun") - assert env.sos_dict["b"] == True + assert env.sos_dict["b"] is True env.sos_dict.pop("b") Base_Executor(wf, args=["--no-b"]).run(mode="dryrun") - assert env.sos_dict["b"] == False + assert env.sos_dict["b"] is False env.sos_dict.pop("b") # bool with default False script = SoS_Script(""" @@ -359,13 +359,13 @@ def test_parameters_section(): """) wf = script.workflow() Base_Executor(wf, args=[]).run(mode="dryrun") - assert env.sos_dict["b"] == False + assert env.sos_dict["b"] is False env.sos_dict.pop("b") Base_Executor(wf, args=["--b"]).run(mode="dryrun") - assert env.sos_dict["b"] == True + assert env.sos_dict["b"] is True env.sos_dict.pop("b") Base_Executor(wf, args=["--no-b"]).run(mode="dryrun") - assert env.sos_dict["b"] == False + assert env.sos_dict["b"] is False env.sos_dict.pop("b") # # parameters cannot coincide with a readonly global variable diff --git a/test/test_utils.py b/test/test_utils.py index 6f57d976b..733c0b960 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -15,8 +15,7 @@ from sos.targets import executable, file_target, sos_step, sos_targets # these functions are normally not available but can be imported # using their names for testing purposes -from sos.utils import (WorkflowDict, as_fstring, env, fileMD5, get_logger, - split_fstring, stable_repr) +from sos.utils import (WorkflowDict, as_fstring, env, fileMD5, get_logger, split_fstring, stable_repr) from sos.workflow_executor import Base_Executor, analyze_section @@ -24,18 +23,10 @@ def test_logger(): '''Test logging level''' for verbosity in [0, 1, 2, 3, 4]: env.verbosity = verbosity - get_logger().debug( - 'Verbosity {}:debug message with ``empahsized text`` in between' - .format(env.verbosity)) - get_logger().info( - 'Verbosity {}:info message with ``empahsized text`` in between' - .format(env.verbosity)) - get_logger().warning( - 'Verbosity {}:warning message with ``empahsized text`` in between' - .format(env.verbosity)) - get_logger().error( - 'Verbosity {}:error message with ``empahsized text`` in between' - .format(env.verbosity)) + get_logger().debug(f'Verbosity {env.verbosity}:debug message with ``empahsized text`` in between') + get_logger().info(f'Verbosity {env.verbosity}:info message with ``empahsized text`` in between') + get_logger().warning(f'Verbosity {env.verbosity}:warning message with ``empahsized text`` in between') + get_logger().error(f'Verbosity {env.verbosity}:error message with ``empahsized text`` in between') def test_workflow_dict(): @@ -78,9 +69,7 @@ def test_pattern_match(): }) assert expand_pattern('{b}.txt') == ['file name.txt'] assert expand_pattern('{c}.txt') == ['file1.txt', 'file2.txt', 'file 3.txt'] - assert expand_pattern('{a}_{c}.txt') == [ - '100_file1.txt', '100_file2.txt', '100_file 3.txt' - ] + assert expand_pattern('{a}_{c}.txt') == ['100_file1.txt', '100_file2.txt', '100_file 3.txt'] def test_accessed_vars(): @@ -90,9 +79,7 @@ def test_accessed_vars(): assert accessed_vars('''a = "C"''') == set() assert accessed_vars('''a = "C" + f"{D}"''') == {'D'} assert accessed_vars('''a = 1 + f"{D + 20:f}" ''') == {'D'} - assert accessed_vars( - '''k, "a.txt", "b.txt", par=f(something) ''', - mode='eva') == {'k', 'f', 'something'} + assert accessed_vars('''k, "a.txt", "b.txt", par=f(something) ''', mode='eva') == {'k', 'f', 'something'} # this is a complicated case because the actual variable depends on the # result of an expression... However, in the NO-evaluation case, this is # the best we can do. @@ -127,8 +114,7 @@ def test_text_repr(): # would incorrectly be executed as bat if sys.platform == 'win32': return - for text in ('"""a"""', '"b"', r'"""\na\\nb"""', r"'''a\nb'''", - """ "a'\\"='" """): + for text in ('"""a"""', '"b"', r'"""\na\\nb"""', r"'''a\nb'''", """ "a'\\"='" """): script = SoS_Script(r''' a = 1 run: expand=True @@ -194,8 +180,7 @@ def test_analyze_section(): assert res['changed_vars'] == {'b'} elif section.names[0][1] == '2': assert res['step_input'] == sos_targets() - assert res['step_depends'] == sos_targets('some.txt', - executable('ls')) + assert res['step_depends'] == sos_targets('some.txt', executable('ls')) assert res['step_output'].unspecified() # for_each will not be used for DAG assert res['environ_vars'] == {'b', 'for_each', 'executable'} @@ -248,8 +233,7 @@ def test_split_fstring(): ('hello {a+b } }} world', ['hello ', 'a+b ', ' }} world']), ('hello {a+b:r} }} world', ['hello ', 'a+b:r', ' }} world']), ('hello {{{a+b!r} }} world', ['hello {{', 'a+b!r', ' }} world']), - ('hello {a+b + {1,2}.pop() } }} world', - ['hello ', 'a+b + {1,2}.pop() ', ' }} world']), + ('hello {a+b + {1,2}.pop() } }} world', ['hello ', 'a+b + {1,2}.pop() ', ' }} world']), ]: if pieces is None: with pytest.raises(SyntaxError): @@ -270,16 +254,12 @@ def test_as_fstring(): ('hello {1} world', 'fr"""hello {1} world"""'), ('hello {a+b } }} world', 'fr"""hello {a+b } }} world"""'), ('hello {a+b:r} }} world', 'fr"""hello {a+b:r} }} world"""'), - ('''hello """ \'\'\' {a+b } }} world''', - 'f\'hello """ \\\'\\\'\\\' {a+b } }} world\''), + ('''hello """ \'\'\' {a+b } }} world''', 'f\'hello """ \\\'\\\'\\\' {a+b } }} world\''), ('hello {{{a+b!r} }} world', 'fr"""hello {{{a+b!r} }} world"""'), - ('hello {a+b + {1,2}.pop() } }} world', - 'fr"""hello {a+b + {1,2}.pop() } }} world"""'), - ('''hello {'a'+b !r} }} world''', - 'fr"""hello {\'a\'+b !r} }} world"""'), + ('hello {a+b + {1,2}.pop() } }} world', 'fr"""hello {a+b + {1,2}.pop() } }} world"""'), + ('''hello {'a'+b !r} }} world''', 'fr"""hello {\'a\'+b !r} }} world"""'), ('''hello """ \'\'\' {'a'+"b" + {"c", "D"}.pop() } }} world''', - '\'hello """ \\\'\\\'\\\' {0} }} world\'.format(\'a\'+"b" + {"c", "D"}.pop() )' - ), + '\'hello """ \\\'\\\'\\\' {0} }} world\'.format(\'a\'+"b" + {"c", "D"}.pop() )'), ]: assert as_fstring(string) == fstring @@ -300,8 +280,7 @@ def test_analyze_output_from(): if section.names[0][1] == 1: assert res['step_depends'] == sos_targets(sos_step('B')) if section.names[0][1] == 2: - assert res['step_depends'] == sos_targets( - sos_step('C1'), sos_step('C2')) + assert res['step_depends'] == sos_targets(sos_step('C1'), sos_step('C2')) def test_file_sig(clear_now_and_after): @@ -322,11 +301,12 @@ def test_file_sig(clear_now_and_after): assert not a.validate() + @pytest.mark.parametrize('fsize', [12354, 33554432, 34605213]) def test_file_md5(fsize, temp_factory): '''test save and validate of file signature''' fname = 'test_md5.txt' - temp_factory(fname, size = fsize) + temp_factory(fname, size=fsize) partial_md5, full_md5 = fileMD5(fname, sig_type='both') assert partial_md5 == fileMD5(fname, sig_type='partial')