Merge branch 'master' into issue1535

.appveyor.yml

@@ -22,18 +22,13 @@ install:
   - echo $PATH
   - echo $PYTHON
   # packages required by SoS
-  - pip install fasteners pygments networkx pydot pydotplus nose
-  - pip install entrypoints numpy pandas
-  - pip install docker pyyaml psutil tqdm graphviz imageio pillow entrypoints
-  - pip install pytest coverage codacy-coverage pytest-cov python-coveralls -U
-
+  - pip install -r requirements_dev.txt
   - pip install .
-  - pip install sos-pbs
 
 test_script:
   - cd test
   - sh build_test_docker.sh
-  - sh run_tests.sh
+  - python run_tests.py
 
   #
   #on_finish:

.github/workflows/pytest.yml

@@ -0,0 +1,31 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: pytest
+
+on: [push, pull_request]
+
+jobs:
+  pytest:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements_dev.txt
+        pip install -e .
+    - name: Run tests
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        cd test
+        sh build_test_docker.sh
+        python run_tests.py

requirements_dev.txt

@@ -1,6 +1,19 @@
-pytest
-pytest-cov
 coverage
+fasteners
+jinja2
+nbformat
+networkx!=2.8.3
 nose
-
-
+pandas
+pexpect
+psutil
+ptyprocess
+pydot
+pydotplus
+pyyaml
+pygments
+pytest
+pytest-cov
+pyzmq
+sos-pbs
+tqdm

src/sos/__main__.py

@@ -2187,7 +2187,7 @@ def cmd_remove(args, unknown_args):
         # a special case where all file and runtime signatures are removed.
         # no other options are allowed.
         if sig_files:
-            sig_ids = list(set([x[0] for x in sig_files]))
+            sig_ids = list({x[0] for x in sig_files})
             step_signatures = StepSignatures()
             num_removed_steps = step_signatures.remove_many(sig_ids)
             if not num_removed_steps:

src/sos/actions.py

@@ -33,7 +33,8 @@
 from .parser import SoS_Script
 from .syntax import SOS_ACTION_OPTIONS
 from .targets import executable, file_target, path, paths, sos_targets
-from .utils import (StopInputGroup, TerminateExecution, TimeoutInterProcessLock, env, fileMD5, get_traceback,
+from .utils import (StopInputGroup, TerminateExecution,
+                    TimeoutInterProcessLock, env, fileMD5, get_traceback,
                     load_config_files, short_repr, textMD5, transcribe)
 
 __all__ = [

src/sos/converter.py

@@ -116,7 +116,7 @@ def analyse_text(self, text):
 ]
 
 
-class ScriptToHTMLConverter(object):
+class ScriptToHTMLConverter:
 
     def __init__(self, *args, **kwargs):
         pass

src/sos/dag.py

@@ -76,7 +76,7 @@
 #
 
 
-class SoS_Node(object):
+class SoS_Node:
 
     def __init__(self, step_uuid: str, node_name: str,
                  wf_index: Union[int, None], node_index: Union[int, None],
@@ -277,10 +277,10 @@ def dangling(self, targets: sos_targets):
                     else:
                         missing.add(x)
         else:
-            missing = set([
+            missing = {
                 x for x in self._all_depends_files.keys()
                 if x not in self._all_output_files and not x.target_exists()
-            ])
+            }
         for x in targets:
             if x not in self._all_output_files:
                 if x.target_exists('target'):

src/sos/eval.py

@@ -273,7 +273,7 @@ def _is_expr(expr):
         return False
 
 
-class StatementHash(object):
+class StatementHash:
     stmt_hash = {}
 
     def __init__(self) -> None:
@@ -391,7 +391,7 @@ def SoS_exec(script: str,
 #
 
 
-class Undetermined(object):
+class Undetermined:
 
     def __init__(self, expr: str = "") -> None:
         if not isinstance(expr, str):
@@ -414,7 +414,7 @@ def targets(self) -> "Undetermined":
         return self
 
 
-class on_demand_options(object):
+class on_demand_options:
     """Expression that will be evaluated upon request."""
 
     def __init__(self, items: Optional[Dict[str, Any]]) -> None:

src/sos/executor_utils.py

@@ -12,9 +12,9 @@
 import traceback
 from collections.abc import Sequence
 from io import StringIO
+from secrets import token_hex
 from tokenize import generate_tokens
 from typing import Any
-from secrets import token_hex
 
 import psutil
 

src/sos/hosts.py

@@ -92,7 +92,7 @@ def run(self):
         sys.stdout.flush()
         sys.stderr.flush()
         try:
-            si = open(os.devnull, "r")
+            si = open(os.devnull)
             so = open(os.devnull, "w")
             se = open(os.devnull, "w")
             os.dup2(si.fileno(), sys.stdin.fileno())
@@ -127,7 +127,7 @@ def _show_err_and_out(task_id, res) -> None:
             sys.stderr.write("\n")
 
 
-class LocalHost(object):
+class LocalHost:
     """For local host, no path map, send and receive ..."""
 
     def __init__(
@@ -291,7 +291,7 @@ def receive_result(self, task_id: str) -> Dict[str, Any]:
         return res
 
 
-class RemoteHost(object):
+class RemoteHost:
     """A remote host class that manages how to communicate with remote host"""
 
     def __init__(

src/sos/parser.py

@@ -87,13 +87,13 @@ def is_type_hint(stmt: str) -> bool:
     # input: variable
     #
     if "=" not in stmt:
-        action, par = [x.strip() for x in stmt.split(":", 1)]
+        action, par = (x.strip() for x in stmt.split(":", 1))
     else:
         # one parameter?
         #
         # action: input={'a': b}
         #
-        action, par = [x.strip() for x in stmt.split("=", 1)[0].split(":", 1)]
+        action, par = (x.strip() for x in stmt.split("=", 1)[0].split(":", 1))
 
     if action in SOS_DIRECTIVES:
         return False

src/sos/pattern.py

@@ -141,7 +141,7 @@ def expand_pattern(pattern: str) -> List[str]:
         if key not in env.sos_dict:
             raise ValueError(f"Undefined variable {key} in pattern {pattern}")
         if not isinstance(env.sos_dict[key], str) and isinstance(
-                env.sos_dict[key], collections.Sequence):
+                env.sos_dict[key], collections.abc.Sequence):
             if sz is None:
                 sz = len(env.sos_dict[key])
                 wildcard = [copy.deepcopy(wildcard[0]) for x in range(sz)]

src/sos/preview.py

@@ -104,7 +104,7 @@ def preview_img(filename, kernel=None, style=None):
 
 
 def preview_svg(filename, kernel=None, style=None):
-    with open(filename, "r") as f:
+    with open(filename) as f:
         image_data = f.read()
     return {"image/svg+xml": image_data}
 
@@ -255,7 +255,7 @@ def preview_txt(filename, kernel=None, style=None):
     hint = f' ({limit} displayed, see --limit)' if nlines > limit else ''
     content = f"HINT: {nlines} line{'s' if nlines > 1 else ''}{hint}\n"
 
-    with open(filename, "r") as fin:
+    with open(filename) as fin:
         if limit < 0:
             content += fin.read()
         else:

src/sos/remote.py

@@ -241,7 +241,7 @@ def test_paths(host):
                 f"Failed to receive file from remote host {remote}: file does not exist"
             )
         # check file content?
-        with open(os.path.join(local, f".sos_test_{tID}.txt"), "r") as tFile:
+        with open(os.path.join(local, f".sos_test_{tID}.txt")) as tFile:
             remote_content = tFile.read()
         if remote_content != str(tID):
             return f"Content of received file does not match: {tID} expected, {remote_content} received."

src/sos/signatures.py

@@ -213,4 +213,3 @@ def clear(self):
         except sqlite3.DatabaseError as e:
             env.logger.warning(f"Failed to clear workflow database: {e}")
             return []
-

src/sos/step_executor.py

@@ -17,16 +17,20 @@
 
 from .controller import close_socket, create_socket, send_message_to_controller
 from .eval import KeepOnlyImportAndDefine, SoS_eval, SoS_exec, accessed_vars
-from .executor_utils import (ExecuteError, __named_output__, __null_func__, __output_from__, __traced__, clear_output,
-                             create_task, get_traceback_msg, reevaluate_output, statementMD5, validate_step_sig,
-                             verify_input)
+from .executor_utils import (ExecuteError, __named_output__, __null_func__,
+                             __output_from__, __traced__, clear_output,
+                             create_task, get_traceback_msg, reevaluate_output,
+                             statementMD5, validate_step_sig, verify_input)
 from .messages import decode_msg, encode_msg
-from .syntax import (SOS_DEPENDS_OPTIONS, SOS_INPUT_OPTIONS, SOS_OUTPUT_OPTIONS, SOS_TARGETS_OPTIONS)
-from .targets import (RemovedTarget, RuntimeInfo, UnavailableLock, UnknownTarget, dynamic, file_target, invalid_target,
+from .syntax import (SOS_DEPENDS_OPTIONS, SOS_INPUT_OPTIONS,
+                     SOS_OUTPUT_OPTIONS, SOS_TARGETS_OPTIONS)
+from .targets import (RemovedTarget, RuntimeInfo, UnavailableLock,
+                      UnknownTarget, dynamic, file_target, invalid_target,
                       sos_step, sos_targets, sos_variable)
 from .tasks import MasterTaskParams, TaskFile
-from .utils import (ArgumentError, ProcessKilled, StopInputGroup, TerminateExecution, env, get_localhost_ip,
-                    get_traceback, short_repr, textMD5)
+from .utils import (ArgumentError, ProcessKilled, StopInputGroup,
+                    TerminateExecution, env, get_localhost_ip, get_traceback,
+                    short_repr, textMD5)
 
 __all__: List = []
 
@@ -510,8 +514,8 @@ def process_output_args(self, ofiles: sos_targets, **kwargs):
 
         # create directory
         if ofiles.valid():
-            parents = set(
-                [os.path.abspath(os.path.join(ofile, os.pardir)) for ofile in ofiles if isinstance(ofile, file_target)])
+            parents = {
+                os.path.abspath(os.path.join(ofile, os.pardir)) for ofile in ofiles if isinstance(ofile, file_target)}
             for parent_dir in parents:
                 if parent_dir and not os.path.isdir(parent_dir):
                     os.makedirs(parent_dir, exist_ok=True)

src/sos/syntax.py

@@ -99,7 +99,7 @@
 #
 
 
-class LazyRegex(object):
+class LazyRegex:
     """A proxy around a real regex, which won't be compiled until accessed."""
 
     # These are the parameters on a real _sre.SRE_Pattern object, which we

src/sos/targets.py

@@ -86,7 +86,7 @@ def __init__(self, signature):
 #
 
 
-class BaseTarget(object):
+class BaseTarget:
     """A base class for all targets (e.g. a file)"""
 
     def __init__(self, *args, **kwargs):
@@ -604,7 +604,7 @@ def validate(self, sig=None):
             md5_file = self + '.md5'
             if md5_file.exists():
                 # validate against md5
-                with open(md5_file, 'r') as mfile:
+                with open(md5_file) as mfile:
                     return mfile.readline().strip().split()[-1] == fileMD5(self, sig_type='full')
         if sig is not None:
             sig_mtime, sig_size, sig_md5 = sig

src/sos/targets_python.py

@@ -24,9 +24,9 @@ def __init__(self, module, version=None, autoinstall=False):
                     raise ValueError(
                         f"Specifying 'version=' option in addition to '{module}' is not allowed"
                     )
-                self._module, self._version = [
+                self._module, self._version = (
                     x.strip() for x in self._module.split(opt, 1)
-                ]
+                )
                 if ',' in self._version:
                     raise ValueError(
                         f'SoS does not yet support multiple version comparisons. {self._mdoule} provided'

src/sos/targets_r.py

@@ -56,7 +56,7 @@ def _install(self, name, version, repos):
                     raise ValueError(
                         f"Specifying 'version=' option in addition to '{name}' is not allowed"
                     )
-                name, version = [x.strip() for x in name.split(opt, 1)]
+                name, version = (x.strip() for x in name.split(opt, 1))
                 if "," in version:
                     raise ValueError(
                         f"SoS does not yet support multiple version comparisons. {version} provided"

src/sos/task_executor.py

@@ -15,23 +15,29 @@
 
 import zmq
 
-from .controller import (Controller, close_socket, connect_controllers, create_socket, disconnect_controllers,
-                         request_answer_from_controller, send_message_to_controller)
+from .controller import (Controller, close_socket, connect_controllers,
+                         create_socket, disconnect_controllers,
+                         request_answer_from_controller,
+                         send_message_to_controller)
 from .eval import SoS_eval, SoS_exec
-from .executor_utils import (__null_func__, clear_output, get_traceback_msg, prepare_env)
+from .executor_utils import (__null_func__, clear_output, get_traceback_msg,
+                             prepare_env)
 from .messages import decode_msg
 from .monitor import TaskMonitor
 from .step_executor import parse_shared_vars
-from .targets import (InMemorySignature, dynamic, file_target, path, sos_step, sos_targets)
-from .tasks import (TaskFile, combine_results, monitor_interval, remove_task_files, resource_monitor_interval)
-from .utils import (ProcessKilled, StopInputGroup, env, get_localhost_ip, pickleable)
+from .targets import (InMemorySignature, dynamic, file_target, path, sos_step,
+                      sos_targets)
+from .tasks import (TaskFile, combine_results, monitor_interval,
+                    remove_task_files, resource_monitor_interval)
+from .utils import (ProcessKilled, StopInputGroup, env, get_localhost_ip,
+                    pickleable)
 
 
 def signal_handler(*args, **kwargs):
     raise ProcessKilled()
 
 
-class BaseTaskExecutor(object):
+class BaseTaskExecutor:
     """Task executor used to execute specified tasks. Any customized task executor
     should derive from this class.
     """
@@ -262,7 +268,7 @@ def execute_single_task(self, task_id, params, runtime, sig_content, quiet=False
                     if not os.path.isfile(logfile):
                         raise ValueError(f"logfile {logfile} does not exist after the completion of task")
                     try:
-                        with open(logfile, "r") as log:
+                        with open(logfile) as log:
                             my_stdout.write(f"logfile: {logfile}\n")
                             my_stdout.write(log.read())
                     except Exception as e: