adding unittest-style tests, bumping version

biosimulators_test_suite/_version.py

@@ -1 +1 @@
-__version__ = '0.0.1'
+__version__ = '0.1.0'

biosimulators_test_suite/data_model.py

@@ -21,16 +21,19 @@ class AbstractTestCase(abc.ABC):
     Attributes:
         id (:obj:`str`): id
         name (:obj:`str`): name
+        description (:obj:`str`): description
     """
 
-    def __init__(self, id=None, name=None):
+    def __init__(self, id=None, name=None, description=None):
         """
         Args:
             id (:obj:`str`, optional): id
             name (:obj:`str`, optional): name
+            description (:obj:`str`): description
         """
         self.id = id
         self.name = name
+        self.description = description
 
     @abc.abstractmethod
     def eval(self, specifications):
@@ -45,15 +48,6 @@ def eval(self, specifications):
         """
         pass  # pragma: no cover
 
-    @abc.abstractmethod
-    def get_description(self):
-        """ Get a description of the case
-
-        Returns:
-            :obj:`str`: description of the case
-        """
-        pass  # pragma: no cover
-
 
 class SedTaskRequirements(object):
     """ Required model format for simulation algorithm for each task in a SED document

biosimulators_test_suite/exec_core.py

@@ -8,7 +8,8 @@
 
 from .data_model import (AbstractTestCase, TestCaseResult,  # noqa: F401
                          TestCaseResultType, SkippedTestCaseException, IgnoreTestCaseWarning)
-from .test_case.combine_archive import CuratedCombineArchiveTestCase
+from .test_case import combine_archive
+from .test_case import docker_image
 import biosimulators_utils.simulator.io
 import capturer
 import datetime
@@ -48,13 +49,51 @@ def find_cases(cls, ids=None):
         Returns:
             :obj:`list` of :obj:`AbstractTestCase`: test cases
         """
-        cases = CuratedCombineArchiveTestCase.find_cases(ids=ids)
+        cases = []
 
+        # get SED-ML cases
+        cases.extend(cls.find_cases_in_module(docker_image, ids=ids))
+
+        # get curated COMBINE/OMEX archive cases
+        cases.extend(combine_archive.find_cases(ids=ids))
+
+        # warn if desired cases weren't found
         if ids is not None:
             missing_ids = set(ids).difference(set(case.id for case in cases))
             if missing_ids:
                 warnings.warn('Some test case(s) were not found:\n  {}'.format('\n  '.join(sorted(missing_ids))), IgnoreTestCaseWarning)
 
+        # return discovered cases
+        return cases
+
+    @classmethod
+    def find_cases_in_module(cls, module, ids=None):
+        """ Discover test cases in a module
+
+        Args:
+            module (:obj:`types.ModuleType`): module
+            ids (:obj:`list` of :obj:`str`, optional): List of ids of test cases to verify. If :obj:`ids`
+                is none, all test cases are verified.
+
+        Returns:
+            :obj:`list` of :obj:`AbstractTestCase`: test cases
+        """
+        cases = []
+        ignored_ids = []
+        module_name = module.__name__.rpartition('.')[2]
+        for child_name in dir(module):
+            child = getattr(module, child_name)
+            if isinstance(child, type) and issubclass(child, AbstractTestCase) and child != AbstractTestCase:
+                print(child)
+                id = module_name + '/' + child_name
+                if ids is None or id in ids:
+                    cases.append(child(id=id))
+                else:
+                    ignored_ids.append(id)
+
+        if ignored_ids:
+            warnings.warn('Some test case(s) were ignored:\n  {}'.format('\n  '.join(sorted(ignored_ids))), IgnoreTestCaseWarning)
+
         return cases
 
     def run(self, specifications):
@@ -89,7 +128,7 @@ def eval_case(self, specifications, case):
         """ Evaluate a test case for a simulator
 
         Args:
-            specifications (:obj:`str` or :obj:`dict`): path or URL to the specifications of the simulator or the specifications of the simulator
+            specifications (:obj:`dict`): specifications of the simulator
             case (:obj:`AbstractTestCase`): test case
 
         Returns:
@@ -131,11 +170,18 @@ def summarize_results(results):
         failure_details = []
         for result in sorted(results, key=lambda result: result.case.id):
             if result.type == TestCaseResultType.passed:
-                result_str = '  * {} ({}, {:.3f} s)\n'.format(result.case.get_description(), result.case.id, result.duration)
+                if result.case.description:
+                    result_str = '  * {}: {} ({:.3f} s)\n'.format(result.case.id, result.case.description, result.duration)
+                else:
+                    result_str = '  * {} ({:.3f} s)\n'.format(result.case.id, result.duration)
                 passed.append(result_str)
 
             elif result.type == TestCaseResultType.failed:
-                result_str = '* {}: {} ({}, {:.3f} s)\n'.format(result.case.get_description(), result.case.id,
+                if result.case.description:
+                    result_str = '* {}: {} ({}, {:.3f} s)\n'.format(result.case.id, result.case.description,
+                                                                    result.exception.__class__.__name__, result.duration)
+                else:
+                    result_str = '* {} ({}, {:.3f} s)\n'.format(result.case.id,
                                                                 result.exception.__class__.__name__, result.duration)
                 failed.append('  ' + result_str)
                 failure_details.append(result_str + '    ```\n    {}\n\n    {}\n    ```\n'.format(

biosimulators_test_suite/test_case/combine_archive.py

@@ -25,6 +25,8 @@
 
 __all__ = ['CuratedCombineArchiveTestCase']
 
+EXAMPLES_DIR = os.path.join(os.path.dirname(__file__), '..', '..', 'examples')
+
 
 class CuratedCombineArchiveTestCase(AbstractTestCase):
     """ A test case for validating a simulator that involves executing a COMBINE/OMEX archive
@@ -39,8 +41,6 @@ class CuratedCombineArchiveTestCase(AbstractTestCase):
         expected_plots (:obj:`list` of :obj:`ExpectedSedPlot`): list of plots expected to be produced by algorithm
     """
 
-    EXAMPLES_DIR = os.path.join(os.path.dirname(__file__), '..', '..', 'examples')
-
     def __init__(self, id=None, name=None, filename=None, task_requirements=None, expected_reports=None, expected_plots=None,
                  assert_no_extra_reports=False, assert_no_extra_datasets=False, assert_no_missing_plots=False, assert_no_extra_plots=False,
                  r_tol=1e-4, a_tol=0.):
@@ -66,6 +66,8 @@ def __init__(self, id=None, name=None, filename=None, task_requirements=None, ex
         self.expected_reports = expected_reports or []
         self.expected_plots = expected_plots or []
 
+        self.description = self.get_description()
+
         self.assert_no_extra_reports = assert_no_extra_reports
         self.assert_no_extra_datasets = assert_no_extra_datasets
         self.assert_no_missing_plots = assert_no_missing_plots
@@ -86,40 +88,6 @@ def get_description(self):
             simulation_algorithms.add(req.simulation_algorithm)
         return '{} / {}'.format(', '.join(sorted(model_formats)), ', '.join(sorted(simulation_algorithms)))
 
-    @classmethod
-    def find_cases(cls, dir_name=None, ids=None):
-        """ Collect test cases from a directory
-
-        Args:
-            dir_name (:obj:`str`, optional): path to find example COMBINE/OMEX archives
-            id (:obj:`list` of :obj:`str`, optional): List of ids of test cases to verify. If :obj:`ids`
-                is none, all test cases are verified.
-
-        Returns:
-            :obj:`list` of :obj:`CuratedCombineArchiveTestCase`: test cases
-        """
-        if dir_name is None:
-            dir_name = cls.EXAMPLES_DIR
-        if not os.path.isdir(dir_name):
-            warnings.warn('Directory of example COMBINE/OMEX archives is not available', IgnoreTestCaseWarning)
-
-        cases = []
-        ignored_ids = set()
-        for md_filename in glob.glob(os.path.join(dir_name, '**/*.json'), recursive=True):
-            rel_filename = os.path.relpath(md_filename, dir_name)
-            id = os.path.splitext(rel_filename)[0]
-            if ids is None or id in ids:
-                case = CuratedCombineArchiveTestCase().from_json(dir_name, rel_filename)
-                cases.append(case)
-            else:
-                ignored_ids.add(id)
-
-        if ignored_ids:
-            warnings.warn('Some test case(s) were ignored:\n  {}'.format('\n  '.join(sorted(ignored_ids))), IgnoreTestCaseWarning)
-
-        # return cases
-        return cases
-
     def from_json(self, base_path, filename):
         """ Read test case from JSON file
 
@@ -326,3 +294,37 @@ def eval(self, specifications):
 
         finally:
             shutil.rmtree(out_dir)
+
+
+def find_cases(dir_name=None, ids=None):
+    """ Collect test cases from a directory
+
+    Args:
+        dir_name (:obj:`str`, optional): path to find example COMBINE/OMEX archives
+        id (:obj:`list` of :obj:`str`, optional): List of ids of test cases to verify. If :obj:`ids`
+            is none, all test cases are verified.
+
+    Returns:
+        :obj:`list` of :obj:`CuratedCombineArchiveTestCase`: test cases
+    """
+    if dir_name is None:
+        dir_name = EXAMPLES_DIR
+    if not os.path.isdir(dir_name):
+        warnings.warn('Directory of example COMBINE/OMEX archives is not available', IgnoreTestCaseWarning)
+
+    cases = []
+    ignored_ids = set()
+    for md_filename in glob.glob(os.path.join(dir_name, '**/*.json'), recursive=True):
+        rel_filename = os.path.relpath(md_filename, dir_name)
+        id = os.path.splitext(rel_filename)[0]
+        if ids is None or id in ids:
+            case = CuratedCombineArchiveTestCase().from_json(dir_name, rel_filename)
+            cases.append(case)
+        else:
+            ignored_ids.add(id)
+
+    if ignored_ids:
+        warnings.warn('Some test case(s) were ignored:\n  {}'.format('\n  '.join(sorted(ignored_ids))), IgnoreTestCaseWarning)
+
+    # return cases
+    return cases

biosimulators_test_suite/test_case/docker_image.py

@@ -0,0 +1,43 @@
+""" Methods for test cases involving checking Docker images
+
+:Author: Jonathan Karr <[email protected]>
+:Date: 2020-12-21
+:Copyright: 2020, Center for Reproducible Biomedical Modeling
+:License: MIT
+"""
+
+from ..data_model import AbstractTestCase
+import docker
+import warnings
+
+
+class OciLabelsCase(AbstractTestCase):
+    """ Test that a Docker image has Open Container Initiative (OCI) labels with metadata about image """
+    EXPECTED_LABELS = [
+        'org.opencontainers.image.authors',
+        'org.opencontainers.image.description',
+        'org.opencontainers.image.documentation',
+        'org.opencontainers.image.licenses',
+        'org.opencontainers.image.revision',
+        'org.opencontainers.image.source',
+        'org.opencontainers.image.title',
+        'org.opencontainers.image.url',
+        'org.opencontainers.image.vendor',
+        'org.opencontainers.image.version',
+    ]
+
+    def eval(self, specifications):
+        """ Evaluate a simulator's performance on a test case
+
+        Args:
+            specifications (:obj:`dict`): specifications of the simulator to validate
+
+        Raises:
+            :obj:`Exception`: if the simulator did not pass the test case
+        """
+        docker_client = docker.from_env()
+        image = docker_client.images.pull(specifications['image']['url'])
+        missing_labels = set(self.EXPECTED_LABELS).difference(set(image.labels.keys()))
+        if missing_labels:
+            warnings.warn('The Docker image should have the following Open Container Initiative (OCI) labels:\n  {}'.format(
+                '\n  '.join(sorted(missing_labels))))

requirements.txt

@@ -1,4 +1,5 @@
 biosimulators_utils[docker] >= 0.1.19
 capturer
 cement
+docker
 natsort

tests/test_case/test_combine_archive.py

@@ -1,5 +1,5 @@
 from biosimulators_test_suite import data_model
-from biosimulators_test_suite.test_case.combine_archive import CuratedCombineArchiveTestCase
+from biosimulators_test_suite.test_case.combine_archive import CuratedCombineArchiveTestCase, find_cases
 from biosimulators_utils.archive.data_model import Archive, ArchiveFile
 from biosimulators_utils.archive.io import ArchiveWriter
 from biosimulators_utils.report.io import ReportWriter, ReportFormat
@@ -16,7 +16,7 @@
 
 class TestCuratedCombineArchiveTestCase(unittest.TestCase):
     def test_find_cases(self):
-        cases = CuratedCombineArchiveTestCase.find_cases(ids=[
+        cases = find_cases(ids=[
             'sbml-core/Caravagna-J-Theor-Biol-2010-tumor-suppressive-oscillations',
             'sbml-core/Ciliberto-J-Cell-Biol-2003-morphogenesis-checkpoint',
         ])
@@ -27,14 +27,14 @@ def test_find_cases(self):
         ]))
 
         with self.assertWarnsRegex(data_model.IgnoreTestCaseWarning, 'archives is not available'):
-            CuratedCombineArchiveTestCase.find_cases(dir_name='does-not-exist')
+            find_cases(dir_name='does-not-exist')
 
-    def test_CuratedCombineArchiveTestCase_get_description(self):
+    def test_CuratedCombineArchiveTestCase_description(self):
         case = CuratedCombineArchiveTestCase(task_requirements=[
             data_model.SedTaskRequirements(model_format='format_2585', simulation_algorithm='KISAO_0000027'),
             data_model.SedTaskRequirements(model_format='format_2585', simulation_algorithm='KISAO_0000019'),
         ])
-        self.assertEqual(case.get_description(), 'format_2585 / KISAO_0000019, KISAO_0000027')
+        self.assertEqual(case.description, 'format_2585 / KISAO_0000019, KISAO_0000027')
 
     def test_CuratedCombineArchiveTestCase_from_dict(self):
         base_path = os.path.join(os.path.dirname(__file__), '..', '..', 'examples')

tests/test_case/test_docker_image.py

@@ -0,0 +1,10 @@
+from biosimulators_test_suite.test_case import docker_image
+import unittest
+
+
+class DockerImageTestCaseTest(unittest.TestCase):
+    def test_OciLabelsCase(self):
+        case = docker_image.OciLabelsCase()
+        case.eval({'image': {'url': 'ghcr.io/biosimulators/biosimulators_copasi/copasi:latest'}})
+        with self.assertWarnsRegex(UserWarning, 'should have the following'):
+            case.eval({'image': {'url': 'hello-world'}})