Merge branch 'dev' into deploy

CONTRIBUTING.md

@@ -56,10 +56,15 @@ Below are instructions for releasing a new version:
 1. Commit the changes to this repository.
 2. Increment the `__version__` variable in `biosimulators_test_suite/_version.py`.
 3. Commit this change to `biosimulators_test_suite/_version.py`.
-4. Add a tag for the new version by running `git tag { version }`. `version` should be equal to the value of the
+4. Merge the changes in to the `deploy` branch:
+    ```
+    git checkout deploy
+    git merge dev
+    ```
+5. Add a tag for the new version by running `git tag { version }`. `version` should be equal to the value of the
    `__version__` variable in `biosimulators_test_suite/_version.py`.
-5. Push these commits and the new tag to GitHub by running `git push && git push --tags`.
-6. This push will trigger a GitHub action which will execute the following tasks:
+6. Push these commits and the new tag to GitHub by running `git push && git push --tags`.
+7. This push will trigger a GitHub action which will execute the following tasks:
    * Create a GitHub release for the version.
    * Push the release to PyPI.
    * Compile the documentation and push the compiled documentation to the repository so that the new documentation is viewable at github.io.

biosimulators_test_suite/_version.py

@@ -1 +1 @@
-__version__ = '0.1.2'
+__version__ = '0.1.3'

biosimulators_test_suite/config.py

@@ -0,0 +1,20 @@
+""" Configuration
+
+:Author: Jonathan Karr <[email protected]>
+:Date: 2020-12-28
+:Copyright: 2020, Center for Reproducible Biomedical Modeling
+:License: MIT
+"""
+
+__all__ = ['TERMINAL_COLORS']
+
+TERMINAL_COLORS = {
+    'pass': 'green',
+    'passed': 'green',
+    'failure': 'red',
+    'failed': 'red',
+    'skip': 'magenta',
+    'skipped': 'magenta',
+    'warning': 'yellow',
+    'warned': 'yellow',
+}

biosimulators_test_suite/data_model.py

@@ -9,15 +9,15 @@
 import abc
 import enum
 
-__all__ = ['AbstractTestCase', 'SedTaskRequirements', 'ExpectedSedReport', 'ExpectedSedPlot',
+__all__ = ['TestCase', 'SedTaskRequirements', 'ExpectedSedReport', 'ExpectedSedPlot',
            'TestCaseResultType', 'TestCaseResult',
            'AlertType',
-           'InvalidOuputsException', 'InvalidOuputsWarning',
+           'TestCaseWarning', 'InvalidOuputsException', 'InvalidOuputsWarning',
            'SkippedTestCaseException', 'IgnoredTestCaseWarning',
            ]
 
 
-class AbstractTestCase(abc.ABC):
+class TestCase(abc.ABC):
     """ A test case for validating a simulator
 
     Attributes:
@@ -123,22 +123,25 @@ class TestCaseResult(object):
         type (:obj:`obj:`TestCaseResultType`): type
         duration (:obj:`float`): execution duration in seconds
         exception (:obj:`Exception`): exception
+        warnings (:obj:`list` of :obj:`TestCaseWarning`): warnings
         log (:obj:`str`): log of execution
     """
 
-    def __init__(self, case=None, type=None, duration=None, exception=None, log=None):
+    def __init__(self, case=None, type=None, duration=None, exception=None, warnings=None, log=None):
         """
         Args:
             case (:obj:`TestCase`, optional): test case
             type (:obj:`obj:`TestCaseResultType`, optional): type
             duration (:obj:`float`, optional): execution duration in seconds
             exception (:obj:`Exception`, optional): exception
+            warnings (:obj:`list` of :obj:`TestCaseWarning`): warnings
             log (:obj:`str`, optional): log of execution
         """
         self.case = case
         self.type = type
         self.duration = duration
         self.exception = exception
+        self.warnings = warnings or []
         self.log = log
 
 
@@ -148,12 +151,17 @@ class AlertType(str, enum.Enum):
     warning = 'warning'
 
 
+class TestCaseWarning(UserWarning):
+    """ Base class for warnings collected from test cases """
+    pass
+
+
 class InvalidOuputsException(Exception):
     """ Exception raised when outputs of execution of COMBINE/OMEX archive are not as expected """
     pass  # pragma: no cover
 
 
-class InvalidOuputsWarning(UserWarning):
+class InvalidOuputsWarning(TestCaseWarning):
     """ Warning raised when outputs of execution of COMBINE/OMEX archive are not as expected """
     pass  # pragma: no cover
 

biosimulators_test_suite/exec_cli.py

@@ -6,9 +6,12 @@
 :License: MIT
 """
 
-from biosimulators_test_suite.data_model import TestCaseResultType
+from .config import TERMINAL_COLORS
+from .data_model import TestCaseResultType
+import biosimulators_test_suite
 import biosimulators_test_suite.exec_core
 import cement
+import termcolor
 
 
 class BaseController(cement.Controller):
@@ -49,15 +52,27 @@ def _default(self):
         args = self.app.pargs
         try:
             validator = biosimulators_test_suite.exec_core.SimulatorValidator(
+                args.specifications,
                 case_ids=args.case_ids,
                 verbose=args.verbose)
-            results = validator.run(args.specifications)
-            summary, failure_details = validator.summarize_results(results)
+            results = validator.run()
+            summary, failure_details, warning_details = validator.summarize_results(results)
+            print('')
             print('=============== SUMMARY ===============')
-            print(summary + '\n')
+            print('')
+            print(summary + '\n\n')
             if failure_details:
-                print('=============== FAILURES ===============')
-                print(failure_details)
+                color = TERMINAL_COLORS['failure']
+                print(termcolor.colored('=============== FAILURES ===============', color))
+                print(termcolor.colored('', color))
+                print(termcolor.colored('* ' + '\n\n* '.join(failure_details), color))
+                print('')
+            if warning_details:
+                color = TERMINAL_COLORS['warning']
+                print(termcolor.colored('=============== WARNINGS ===============', color))
+                print(termcolor.colored('', color))
+                print(termcolor.colored('* ' + '\n\n* '.join(warning_details), color))
+                print('')
         except Exception as exception:
             raise SystemExit(str(exception))