Merge branch 'master' into plumbing

* master: (61 commits)
  prepare 8.0a2 release
  CYLC_CONF_PATH: modify interpretation * if specified it must exist * if it's not readable you get and error because thats your fault * it can specify a file or a directory * also use pathlib because it's pretty
  scheduler_cli: add asyncio notes
  tests: fix version dependent test
  prepare 8.0a2 release
  tests: fix tests/f/cylc-ping/03
  scheduler: shutdown logic for paritally initiated flows
  tests/i: tidy test utilities away out of sight
  pytest: exclude un-collectable test file
  scheduler_cli: simplify cli teardown
  tests/i: simplify teardown
  pycodestyle++
  tests/i: suppress NFS teardown issues
  tests/i: fix asyncio event loop issues
  asyncio: shut down workflow tasks cleanly
  tests/i: fix nfs issues with integration testing cleanup/teardown
  tests/i: forward port test changes from cylc#3500
  tests/i: clean tests
  tests: documentation
  setup: add missing test dependency
  ...

.codacy.yml

@@ -2,6 +2,5 @@ exclude_paths:
   - 'conf/**'
   - 'doc/joss-paper/**'
   - 'etc/**'
-  - 'cylc/flow/tests/**'
   - 'tests/**'
   - 'cylc/flow/**_pb2.py'

.codecov.yml

@@ -40,7 +40,7 @@ coverage:
 
 # files to ignore
 ignore:
-  - "tests/**/*.py"
+  - "tests/**"
   - "ws_messages_pb2.py"
 
 # turn off comments to pull requests

.coveragerc

@@ -29,11 +29,10 @@ debug=
 #include
 note=
 omit=
-    */cylc/flow/tests/*
+    tests/*
     */cylc/flow/profiler.py
     */cylc/flow/profiling/*
     */cylc/flow/parsec/OrderedDictCompat.py
-    */cylc/flow/parsec/tests/*
     */cylc/flow/*_pb2.py
 parallel = True
 plugins=
@@ -55,11 +54,10 @@ fail_under=0
 ignore_errors = False
 include=
 omit=
-    */cylc/flow/tests/*
+    tests/*
     */cylc/flow/profiler.py
     */cylc/flow/profiling/*
     */cylc/flow/parsec/OrderedDictCompat.py
-    */cylc/flow/parsec/tests/*
     */cylc/flow/*_pb2.py
 partial_branches=
 precision=2

.github/workflows/test.yml

@@ -4,7 +4,7 @@ on: [pull_request]
 
 
 jobs:
-  unit-test:
+  fast-tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -26,29 +26,32 @@ jobs:
       - name: Install
         run: |
           pip install ."[all]"
-          pip install pytest-xdist
 
       - name: Style
         run: |
-          pycodestyle cylc/flow
+          pycodestyle
           etc/bin/shellchecker
 
-      - name: Unit Test
+      - name: Unit Tests
         run: |
-          pytest -n 5
+          pytest -n 5 tests/unit
+
+      - name: Integration Tests
+        run: |
+          pytest -n 5 tests/integration
 
 
-  functional_test:
+  functional-tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ['3.7']
         tests: [
-          ['tests', '1/4'],
-          ['tests', '2/4'],
-          ['tests', '3/4'],
-          ['tests', '4/4'],
-          ['flakytests', '1/1']
+          ['tests/f', '1/4'],
+          ['tests/f', '2/4'],
+          ['tests/f', '3/4'],
+          ['tests/f', '4/4'],
+          ['tests/k', '1/1']
         ]
     env:
       BASE: ${{ matrix.tests[0] }}
@@ -73,7 +76,7 @@ jobs:
 
       - name: Test
         run: |
-          if [[ "${BASE}" == 'flakytests' ]]; then
+          if [[ "${BASE}" == 'tests/k' ]]; then
               NPROC=4
           else
               NPROC=8

CHANGES.md

@@ -60,7 +60,7 @@ The following obsolete command line options have been removed:
 * `--no-login`
 
 -------------------------------------------------------------------------------
-## __cylc-8.0a2 (2019-Q4?)__
+## __cylc-8.0a2 (2020-07-03)__
 
 Third alpha release of Cylc 8.
 

cylc/flow/__init__.py

@@ -63,4 +63,4 @@ def environ_path_add(dirs, key='PATH'):
 
 environ_init()
 
-__version__ = "8.0a1"
+__version__ = '8.0a2'

cylc/flow/cfgspec/globalcfg.py

@@ -15,6 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Cylc site and user configuration file spec."""
 
+from pathlib import Path
 import os
 import re
 
@@ -723,8 +724,8 @@ class GlobalConfig(ParsecConfig):
     _DEFAULT = None
     _HOME = os.getenv('HOME') or get_user_home()
     CONF_BASENAME = "flow.rc"
-    SITE_CONF_DIR = os.path.join(os.sep, 'etc', 'cylc', 'flow', CYLC_VERSION)
-    USER_CONF_DIR = os.path.join(_HOME, '.cylc', 'flow', CYLC_VERSION)
+    SITE_CONF_DIR = Path(os.sep, 'etc', 'cylc', 'flow', CYLC_VERSION)
+    USER_CONF_DIR = Path(_HOME, '.cylc', 'flow', CYLC_VERSION)
 
     @classmethod
     def get_inst(cls, cached=True):
@@ -753,16 +754,20 @@ def load(self):
         LOG.debug("Loading site/user config files")
         conf_path_str = os.getenv("CYLC_CONF_PATH")
         if conf_path_str:
-            # Explicit config file override.
-            fname = os.path.join(conf_path_str, self.CONF_BASENAME)
-            if os.access(fname, os.F_OK | os.R_OK):
-                self.loadcfg(fname, upgrader.USER_CONFIG)
+            path = Path(conf_path_str)
+            if path.is_dir():
+                path /= self.CONF_BASENAME
+            if not path.exists():
+                # this check is superfluous but it gives us a nicer error
+                raise ValueError(
+                    f'CYLC_CONF_PATH does not exist: {conf_path_str}')
+            self.loadcfg(path, upgrader.USER_CONFIG)
         elif conf_path_str is None:
             # Use default locations.
             for conf_dir, conf_type in [
                     (self.SITE_CONF_DIR, upgrader.SITE_CONFIG),
                     (self.USER_CONF_DIR, upgrader.USER_CONFIG)]:
-                fname = os.path.join(conf_dir, self.CONF_BASENAME)
+                fname = conf_dir / self.CONF_BASENAME
                 if not os.access(fname, os.F_OK | os.R_OK):
                     continue
                 try:

cylc/flow/data_store_mgr.py

@@ -471,7 +471,7 @@ def generate_definition_elements(self):
         workflow.name = self.schd.suite
         workflow.owner = self.schd.owner
         workflow.host = self.schd.host
-        workflow.port = self.schd.port
+        workflow.port = self.schd.port or -1
         for key, val in config.cfg['meta'].items():
             if key in ['title', 'description', 'URL']:
                 setattr(workflow.meta, key, val)

cylc/flow/network/publisher.py

@@ -83,10 +83,13 @@ async def send_multi(self, topic, data, serializer=None):
             serializer (object, optional): string/func for encoding.
 
         """
-        self.topics.add(topic)
-        self.socket.send_multipart(
-            [topic, serialize_data(data, serializer)]
-        )
+        if self.socket:
+            # don't attempt to send anything if we are in the process of
+            # shutting down
+            self.topics.add(topic)
+            self.socket.send_multipart(
+                [topic, serialize_data(data, serializer)]
+            )
 
     async def publish(self, items):
         """Publish topics.

cylc/flow/option_parsers.py

@@ -16,7 +16,7 @@
 """Common options for all cylc commands."""
 
 import logging
-from optparse import OptionParser, OptionConflictError
+from optparse import OptionParser, OptionConflictError, Values
 import os
 import sys
 
@@ -265,3 +265,55 @@ def parse_args(self, remove_opts=None):
         LOG.addHandler(errhandler)
 
         return (options, args)
+
+
+class Options(Values):
+    """Wrapper to allow Python API access to optparse CLI functionality.
+
+    Example:
+        Create an optparse parser as normal:
+        >>> import optparse
+        >>> parser = optparse.OptionParser()
+        >>> _ = parser.add_option('-a', default=1)
+        >>> _ = parser.add_option('-b', default=2)
+
+        Create an Options object from the parser:
+        >>> PythonOptions = Options(parser, overrides={'c': 3})
+
+        "Parse" options via Python API:
+        >>> opts = PythonOptions(a=4)
+
+        Access options as normal:
+        >>> opts.a
+        4
+        >>> opts.b
+        2
+        >>> opts.c
+        3
+
+        Optparse allows you to create new options on the fly:
+        >>> opts.d = 5
+        >>> opts.d
+        5
+
+        But you can't create new options at initiation, this gives us basic
+        input validation:
+        >>> opts(e=6)
+        Traceback (most recent call last):
+        ValueError: e
+
+    """
+
+    def __init__(self, parser, overrides=None):
+        if overrides is None:
+            overrides = {}
+        defaults = {**parser.defaults, **overrides}
+        Values.__init__(self, defaults)
+
+    def __call__(self, **kwargs):
+        for key, value in kwargs.items():
+            if hasattr(self, key):
+                setattr(self, key, value)
+            else:
+                raise ValueError(key)
+        return self

cylc/flow/parsec/fileparse.py

@@ -246,8 +246,8 @@ def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
             try:
                 from cylc.flow.parsec.empysupport import empyprocess
             except (ImportError, ModuleNotFoundError):
-                raise ParsecError('EmPy Python package must be installed '
-                                  'to process file: ' + fpath)
+                raise ParsecError(f'EmPy Python package must be installed '
+                                  'to process file: {fpath}')
             flines = empyprocess(flines, fdir, template_vars)
 
     # process with Jinja2
@@ -257,8 +257,8 @@ def read_and_proc(fpath, template_vars=None, viewcfg=None, asedit=False):
             try:
                 from cylc.flow.parsec.jinja2support import jinja2process
             except (ImportError, ModuleNotFoundError):
-                raise ParsecError('Jinja2 Python package must be installed '
-                                  'to process file: ' + fpath)
+                raise ParsecError(f'Jinja2 Python package must be installed '
+                                  'to process file: {fpath}')
             flines = jinja2process(flines, fdir, template_vars)
 
     # concatenate continuation lines