Code cleanup

src/sos/__main__.py

@@ -2203,7 +2203,7 @@ def cmd_remove(args, unknown_args):
         # a special case where all file and runtime signatures are removed.
         # no other options are allowed.
         if sig_files:
-            sig_ids = list(set([x[0] for x in sig_files]))
+            sig_ids = list({x[0] for x in sig_files})
             step_signatures = StepSignatures()
             num_removed_steps = step_signatures.remove_many(sig_ids)
             if not num_removed_steps:

src/sos/actions.py

@@ -33,7 +33,8 @@
 from .parser import SoS_Script
 from .syntax import SOS_ACTION_OPTIONS
 from .targets import executable, file_target, path, paths, sos_targets
-from .utils import (StopInputGroup, TerminateExecution, TimeoutInterProcessLock, env, fileMD5, get_traceback,
+from .utils import (StopInputGroup, TerminateExecution,
+                    TimeoutInterProcessLock, env, fileMD5, get_traceback,
                     load_config_files, short_repr, textMD5, transcribe)
 
 __all__ = [

src/sos/converter.py

@@ -116,7 +116,7 @@ def analyse_text(self, text):
 ]
 
 
-class ScriptToHTMLConverter(object):
+class ScriptToHTMLConverter:
 
     def __init__(self, *args, **kwargs):
         pass

src/sos/dag.py

@@ -76,7 +76,7 @@
 #
 
 
-class SoS_Node(object):
+class SoS_Node:
 
     def __init__(self, step_uuid: str, node_name: str,
                  wf_index: Union[int, None], node_index: Union[int, None],
@@ -277,10 +277,10 @@ def dangling(self, targets: sos_targets):
                     else:
                         missing.add(x)
         else:
-            missing = set([
+            missing = {
                 x for x in self._all_depends_files.keys()
                 if x not in self._all_output_files and not x.target_exists()
-            ])
+            }
         for x in targets:
             if x not in self._all_output_files:
                 if x.target_exists('target'):

src/sos/eval.py

@@ -273,7 +273,7 @@ def _is_expr(expr):
         return False
 
 
-class StatementHash(object):
+class StatementHash:
     stmt_hash = {}
 
     def __init__(self) -> None:
@@ -391,7 +391,7 @@ def SoS_exec(script: str,
 #
 
 
-class Undetermined(object):
+class Undetermined:
 
     def __init__(self, expr: str = "") -> None:
         if not isinstance(expr, str):
@@ -414,7 +414,7 @@ def targets(self) -> "Undetermined":
         return self
 
 
-class on_demand_options(object):
+class on_demand_options:
     """Expression that will be evaluated upon request."""
 
     def __init__(self, items: Optional[Dict[str, Any]]) -> None:

src/sos/executor_utils.py

@@ -12,9 +12,9 @@
 import traceback
 from collections.abc import Sequence
 from io import StringIO
+from secrets import token_hex
 from tokenize import generate_tokens
 from typing import Any
-from secrets import token_hex
 
 import psutil
 

src/sos/hosts.py

@@ -92,7 +92,7 @@ def run(self):
         sys.stdout.flush()
         sys.stderr.flush()
         try:
-            si = open(os.devnull, "r")
+            si = open(os.devnull)
             so = open(os.devnull, "w")
             se = open(os.devnull, "w")
             os.dup2(si.fileno(), sys.stdin.fileno())
@@ -127,7 +127,7 @@ def _show_err_and_out(task_id, res) -> None:
             sys.stderr.write("\n")
 
 
-class LocalHost(object):
+class LocalHost:
     """For local host, no path map, send and receive ..."""
 
     def __init__(
@@ -297,7 +297,7 @@ def receive_result(self, task_id: str) -> Dict[str, Any]:
         return res
 
 
-class RemoteHost(object):
+class RemoteHost:
     """A remote host class that manages how to communicate with remote host"""
 
     def __init__(

src/sos/parser.py

@@ -87,13 +87,13 @@ def is_type_hint(stmt: str) -> bool:
     # input: variable
     #
     if "=" not in stmt:
-        action, par = [x.strip() for x in stmt.split(":", 1)]
+        action, par = (x.strip() for x in stmt.split(":", 1))
     else:
         # one parameter?
         #
         # action: input={'a': b}
         #
-        action, par = [x.strip() for x in stmt.split("=", 1)[0].split(":", 1)]
+        action, par = (x.strip() for x in stmt.split("=", 1)[0].split(":", 1))
 
     if action in SOS_DIRECTIVES:
         return False

src/sos/pattern.py

@@ -141,7 +141,7 @@ def expand_pattern(pattern: str) -> List[str]:
         if key not in env.sos_dict:
             raise ValueError(f"Undefined variable {key} in pattern {pattern}")
         if not isinstance(env.sos_dict[key], str) and isinstance(
-                env.sos_dict[key], collections.Sequence):
+                env.sos_dict[key], collections.abc.Sequence):
             if sz is None:
                 sz = len(env.sos_dict[key])
                 wildcard = [copy.deepcopy(wildcard[0]) for x in range(sz)]

src/sos/preview.py

@@ -104,7 +104,7 @@ def preview_img(filename, kernel=None, style=None):
 
 
 def preview_svg(filename, kernel=None, style=None):
-    with open(filename, "r") as f:
+    with open(filename) as f:
         image_data = f.read()
     return {"image/svg+xml": image_data}
 
@@ -255,7 +255,7 @@ def preview_txt(filename, kernel=None, style=None):
     hint = f' ({limit} displayed, see --limit)' if nlines > limit else ''
     content = f"HINT: {nlines} line{'s' if nlines > 1 else ''}{hint}\n"
 
-    with open(filename, "r") as fin:
+    with open(filename) as fin:
         if limit < 0:
             content += fin.read()
         else:

src/sos/remote.py

@@ -248,7 +248,7 @@ def test_paths(host):
                 f"Failed to receive file from remote host {remote}: file does not exist"
             )
         # check file content?
-        with open(os.path.join(local, f".sos_test_{tID}.txt"), "r") as tFile:
+        with open(os.path.join(local, f".sos_test_{tID}.txt")) as tFile:
             remote_content = tFile.read()
         if remote_content != str(tID):
             return f"Content of received file does not match: {tID} expected, {remote_content} received."

src/sos/signatures.py

@@ -213,4 +213,3 @@ def clear(self):
         except sqlite3.DatabaseError as e:
             env.logger.warning(f"Failed to clear workflow database: {e}")
             return []
-

src/sos/step_executor.py

@@ -17,16 +17,20 @@
 
 from .controller import close_socket, create_socket, send_message_to_controller
 from .eval import KeepOnlyImportAndDefine, SoS_eval, SoS_exec, accessed_vars
-from .executor_utils import (ExecuteError, __named_output__, __null_func__, __output_from__, __traced__, clear_output,
-                             create_task, get_traceback_msg, reevaluate_output, statementMD5, validate_step_sig,
-                             verify_input)
+from .executor_utils import (ExecuteError, __named_output__, __null_func__,
+                             __output_from__, __traced__, clear_output,
+                             create_task, get_traceback_msg, reevaluate_output,
+                             statementMD5, validate_step_sig, verify_input)
 from .messages import decode_msg, encode_msg
-from .syntax import (SOS_DEPENDS_OPTIONS, SOS_INPUT_OPTIONS, SOS_OUTPUT_OPTIONS, SOS_TARGETS_OPTIONS)
-from .targets import (RemovedTarget, RuntimeInfo, UnavailableLock, UnknownTarget, dynamic, file_target, invalid_target,
+from .syntax import (SOS_DEPENDS_OPTIONS, SOS_INPUT_OPTIONS,
+                     SOS_OUTPUT_OPTIONS, SOS_TARGETS_OPTIONS)
+from .targets import (RemovedTarget, RuntimeInfo, UnavailableLock,
+                      UnknownTarget, dynamic, file_target, invalid_target,
                       sos_step, sos_targets, sos_variable)
 from .tasks import MasterTaskParams, TaskFile
-from .utils import (ArgumentError, ProcessKilled, StopInputGroup, TerminateExecution, env, get_localhost_ip,
-                    get_traceback, short_repr, textMD5)
+from .utils import (ArgumentError, ProcessKilled, StopInputGroup,
+                    TerminateExecution, env, get_localhost_ip, get_traceback,
+                    short_repr, textMD5)
 
 __all__: List = []
 
@@ -510,8 +514,8 @@ def process_output_args(self, ofiles: sos_targets, **kwargs):
 
         # create directory
         if ofiles.valid():
-            parents = set(
-                [os.path.abspath(os.path.join(ofile, os.pardir)) for ofile in ofiles if isinstance(ofile, file_target)])
+            parents = {
+                os.path.abspath(os.path.join(ofile, os.pardir)) for ofile in ofiles if isinstance(ofile, file_target)}
             for parent_dir in parents:
                 if parent_dir and not os.path.isdir(parent_dir):
                     os.makedirs(parent_dir, exist_ok=True)

src/sos/syntax.py

@@ -99,7 +99,7 @@
 #
 
 
-class LazyRegex(object):
+class LazyRegex:
     """A proxy around a real regex, which won't be compiled until accessed."""
 
     # These are the parameters on a real _sre.SRE_Pattern object, which we

src/sos/targets.py

@@ -89,7 +89,7 @@ def __init__(self, signature):
 #
 
 
-class BaseTarget(object):
+class BaseTarget:
     """A base class for all targets (e.g. a file)"""
 
     def __init__(self, *args, **kwargs):
@@ -828,7 +828,7 @@ def validate(self, sig=None):
             md5_file = self + '.md5'
             if md5_file.exists():
                 # validate against md5
-                with open(md5_file, 'r') as mfile:
+                with open(md5_file) as mfile:
                     return mfile.readline().strip().split()[-1] == fileMD5(self, sig_type='full')
         if sig is not None:
             sig_mtime, sig_size, sig_md5 = sig

src/sos/targets_python.py

@@ -24,9 +24,9 @@ def __init__(self, module, version=None, autoinstall=False):
                     raise ValueError(
                         f"Specifying 'version=' option in addition to '{module}' is not allowed"
                     )
-                self._module, self._version = [
+                self._module, self._version = (
                     x.strip() for x in self._module.split(opt, 1)
-                ]
+                )
                 if ',' in self._version:
                     raise ValueError(
                         f'SoS does not yet support multiple version comparisons. {self._mdoule} provided'

src/sos/targets_r.py

@@ -56,7 +56,7 @@ def _install(self, name, version, repos):
                     raise ValueError(
                         f"Specifying 'version=' option in addition to '{name}' is not allowed"
                     )
-                name, version = [x.strip() for x in name.split(opt, 1)]
+                name, version = (x.strip() for x in name.split(opt, 1))
                 if "," in version:
                     raise ValueError(
                         f"SoS does not yet support multiple version comparisons. {version} provided"

src/sos/task_executor.py

@@ -15,23 +15,29 @@
 
 import zmq
 
-from .controller import (Controller, close_socket, connect_controllers, create_socket, disconnect_controllers,
-                         request_answer_from_controller, send_message_to_controller)
+from .controller import (Controller, close_socket, connect_controllers,
+                         create_socket, disconnect_controllers,
+                         request_answer_from_controller,
+                         send_message_to_controller)
 from .eval import SoS_eval, SoS_exec
-from .executor_utils import (__null_func__, clear_output, get_traceback_msg, prepare_env)
+from .executor_utils import (__null_func__, clear_output, get_traceback_msg,
+                             prepare_env)
 from .messages import decode_msg
 from .monitor import TaskMonitor
 from .step_executor import parse_shared_vars
-from .targets import (InMemorySignature, dynamic, file_target, path, sos_step, sos_targets)
-from .tasks import (TaskFile, combine_results, monitor_interval, remove_task_files, resource_monitor_interval)
-from .utils import (ProcessKilled, StopInputGroup, env, get_localhost_ip, pickleable)
+from .targets import (InMemorySignature, dynamic, file_target, path, sos_step,
+                      sos_targets)
+from .tasks import (TaskFile, combine_results, monitor_interval,
+                    remove_task_files, resource_monitor_interval)
+from .utils import (ProcessKilled, StopInputGroup, env, get_localhost_ip,
+                    pickleable)
 
 
 def signal_handler(*args, **kwargs):
     raise ProcessKilled()
 
 
-class BaseTaskExecutor(object):
+class BaseTaskExecutor:
     """Task executor used to execute specified tasks. Any customized task executor
     should derive from this class.
     """
@@ -263,7 +269,7 @@ def execute_single_task(self, task_id, params, runtime, sig_content, quiet=False
                     if not os.path.isfile(logfile):
                         raise ValueError(f"logfile {logfile} does not exist after the completion of task")
                     try:
-                        with open(logfile, "r") as log:
+                        with open(logfile) as log:
                             my_stdout.write(f"logfile: {logfile}\n")
                             my_stdout.write(log.read())
                     except Exception as e:

src/sos/tasks.py

@@ -18,14 +18,15 @@
 import fasteners
 
 from .targets import sos_targets
-from .utils import (DelayedAction, env, expand_size, expand_time, format_duration, format_HHMMSS, linecount_of_file,
+from .utils import (DelayedAction, env, expand_size, expand_time,
+                    format_duration, format_HHMMSS, linecount_of_file,
                     pretty_size, sample_lines, short_repr, tail_of_file)
 
 monitor_interval = 5
 resource_monitor_interval = 60
 
 
-class TaskParams(object):
+class TaskParams:
     """A parameter object that encaptulates parameters sending to
     task executors. This would makes the output of workers, especially
     in the web interface much cleaner (issue #259)"""
@@ -295,7 +296,7 @@ class TaskStatus(Enum):
     completed = 6
 
 
-class TaskFile(object):
+class TaskFile:
     """
     The task file has the following format:
 
@@ -526,7 +527,7 @@ def add_result(self, result: dict = {}):
             params = self._get_params()
             # this is a master task, get all sub task IDs
             if hasattr(params, "task_stack"):
-                missing_tasks = set([x[0] for x in params.task_stack])
+                missing_tasks = {x[0] for x in params.task_stack}
                 #
                 cache_file = os.path.join(os.path.expanduser("~"), ".sos", "tasks", self.task_id + ".cache")
                 results = []
@@ -1725,7 +1726,7 @@ def purge_tasks(tasks, purge_all=None, age=None, status=None, tags=None, verbosi
         all_tasks = [x for x in all_tasks if any(x in tags for x in TaskFile(x[0]).tags.split())]
     #
     # remoe all task files
-    all_tasks = set([x[0] for x in all_tasks])
+    all_tasks = {x[0] for x in all_tasks}
     if all_tasks:
         #
         # find all related files, including those in nested directories