Added task_id to Tests and Test Reports

pytest.ini

@@ -1,3 +1,7 @@
 [pytest]
-norecursedirs = .git .github example* traceback-styles*
-cache_dir = /tmp/python_cache_dir
+norecursedirs =
+        .git .github example* traceback-styles*
+cache_dir =
+        /tmp/python_cache_dir
+markers =
+        task: A concept exercise task.

runner/__init__.py

@@ -23,12 +23,20 @@ def __init__(self):
         self.config = None
 
     def pytest_configure(self, config):
+        config.addinivalue_line("markers", "task(taskno): this marks the exercise task number.")
         self.config = config
 
     def pytest_collection_modifyitems(self, session, config, items):
         """
-        Sorts the tests in definition order.
+        Sorts the tests in definition order & extracts task_id
         """
+        for item in items:
+            test_id = Hierarchy(item.nodeid)
+            name = '.'.join(test_id.split("::")[1:])
+
+            for mark in item.iter_markers(name='task'):
+                self.tests[name] = Test(name=name, task_id=mark.kwargs['taskno'])
+
 
         def _sort_by_lineno(item):
             test_id = Hierarchy(item.nodeid)
@@ -44,9 +52,11 @@ def pytest_runtest_logreport(self, report):
 
         name = report.head_line if report.head_line else ".".join(report.nodeid.split("::")[1:])
 
+
         if name not in self.tests:
             self.tests[name] = Test(name)
 
+
         state = self.tests[name]
 
         # ignore succesful setup and teardown stages
@@ -81,6 +91,19 @@ def pytest_runtest_logreport(self, report):
         source = Path(self.config.rootdir) / report.fspath
         state.test_code = TestOrder.function_source(test_id, source)
 
+        # Looks up tast_ids from parent when the test is a subtest.
+        if state.task_id == 0 and 'variation' in state.name:
+            parent_test_name = state.name.split(' ')[0]
+            parent_task_id = self.tests[parent_test_name].task_id
+            state.task_id = parent_task_id
+
+            # Changes status of parent to fail if any of the subtests fail.
+            if state.fail:
+                self.tests[parent_test_name].fail(
+                    message="One or more subtests for this test failed. Details can be found under each variant."
+                )
+                self.tests[parent_test_name].test_code = state.test_code
+
     def pytest_sessionfinish(self, session, exitstatus):
         """
         Processes the results into a report.

runner/data.py

@@ -49,6 +49,7 @@ class Test:
     status: Status = Status.PASS
     message: Message = None
     test_code: str = ""
+    task_id: int = 0
 
     # for an explanation of why both of these are necessary see
     # https://florimond.dev/blog/articles/2018/10/reconciling-dataclasses-and-properties-in-python/
@@ -57,6 +58,7 @@ class Test:
 
     def _update(self, status: Status, message: Message = None) -> None:
         self.status = status
+
         if message:
             self.message = message
 
@@ -93,6 +95,7 @@ def error(self, message: Message = None) -> None:
         """
         self._update(Status.ERROR, message)
 
+
     def is_passing(self):
         """
         Check if the test is currently passing.
@@ -106,7 +109,7 @@ class Results:
     Overall results of a test run.
     """
 
-    version: int = 2
+    version: int = 3
     status: Status = Status.PASS
     message: Message = None
     tests: List[Test] = field(default_factory=list)
@@ -117,6 +120,7 @@ def add(self, test: Test) -> None:
         """
         if test.status is Status.FAIL:
             self.fail()
+
         self.tests.append(test)
 
     def fail(self) -> None:

runner/sort.py

@@ -57,6 +57,7 @@ def _visit_definition(self, node):
             while isinstance(last_body, (For, While, If)):
                 last_body = last_body.body[-1]
 
+
             testinfo = TestInfo(node.lineno, last_body.lineno, 1)
             self._cache[self.get_hierarchy(Hierarchy(node.name))] = testinfo
 

test/example-all-fail/results.json

@@ -1,30 +1,34 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "fail",
   "tests": [
     {
       "name": "ExampleAllFailTest.test_hello",
       "status": "fail",
       "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!",
-      "test_code": "self.assertEqual(hello(), \"Hello, World!\")"
+      "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 0
     },
     {
       "name": "ExampleAllFailTest.test_abc",
       "status": "fail",
       "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!",
-      "test_code": "self.assertEqual(hello(), \"Hello, World!\")"
+      "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 0
     },
     {
       "name": "ExampleAllFailOtherTest.test_dummy",
       "status": "fail",
       "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!",
-      "test_code": "self.assertEqual(hello(), \"Hello, World!\")"
+      "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 0
     },
     {
       "name": "ExampleAllFailOtherTest.test_hello",
       "status": "fail",
       "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!",
-      "test_code": "self.assertEqual(hello(), \"Hello, World!\")"
+      "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 0
     }
   ]
 }

test/example-empty-file/results.json

@@ -1,5 +1,5 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "error",
   "message": "ImportError: cannot import name 'hello' from 'example_empty_file' (./test/example-empty-file/example_empty_file.py)",
   "tests": []

test/example-has-stdout-and-tasks/example_has_stdout_and_tasks.py

@@ -0,0 +1,21 @@
+"""Example Exercism/Python solution file"""
+
+
+def hello():
+    print("Hello, World!")
+
+
+def must_truncate():
+    print(
+        """
+        Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate sapien. Nibh venenatis cras sed felis. Tortor at risus viverra adipiscing at. Orci dapibus ultrices in iaculis nunc. In fermentum et sollicitudin ac orci phasellus egestas tellus. Tincidunt lobortis feugiat vivamus at augue eget arcu dictum varius. Volutpat est velit egestas dui id. Non nisi est sit amet facilisis magna etiam tempor. Tincidunt id aliquet risus feugiat in ante metus dictum. Viverra aliquet eget sit amet tellus cras adipiscing. Arcu dictum varius duis at. Aliquet lectus proin nibh nisl.
+
+        Urna molestie at elementum eu. Morbi quis commodo odio aenean. Commodo elit at imperdiet dui accumsan sit amet nulla. Faucibus a pellentesque sit amet porttitor. Donec pretium vulputate sapien nec. Felis eget velit aliquet sagittis id consectetur. Nulla malesuada pellentesque elit eget gravida cum. Mauris augue neque gravida in fermentum et sollicitudin. At quis risus sed vulputate odio ut enim blandit volutpat. Enim blandit volutpat maecenas volutpat blandit. Diam in arcu cursus euismod. Congue nisi vitae suscipit tellus mauris a diam.
+
+        Adipiscing enim eu turpis egestas pretium aenean pharetra magna ac. Et odio pellentesque diam volutpat commodo sed egestas. Nulla porttitor massa id neque aliquam vestibulum morbi blandit cursus. Nisi porta lorem mollis aliquam ut porttitor. Morbi leo urna molestie at elementum eu facilisis. Vel elit scelerisque mauris pellentesque pulvinar. Fames ac turpis egestas maecenas pharetra convallis. Ornare arcu dui vivamus arcu felis bibendum ut tristique et. Blandit massa enim nec dui nunc mattis enim. Elit ullamcorper dignissim cras tincidunt lobortis feugiat vivamus at augue.
+
+        Scelerisque varius morbi enim nunc faucibus a pellentesque sit. Enim diam vulputate ut pharetra. Tempor orci eu lobortis elementum nibh tellus molestie nunc non. Cras pulvinar mattis nunc sed. Ac turpis egestas maecenas pharetra convallis posuere morbi leo. Platea dictumst quisque sagittis purus sit amet. Vitae tortor condimentum lacinia quis vel eros donec ac odio. Viverra nibh cras pulvinar mattis nunc sed blandit. Tincidunt lobortis feugiat vivamus at augue. Duis at consectetur lorem donec massa sapien faucibus. Magna ac placerat vestibulum lectus mauris ultrices. Convallis posuere morbi leo urna molestie at.
+
+        Porta non pulvinar neque laoreet suspendisse interdum consectetur libero. Id faucibus nisl tincidunt eget nullam. Ultricies lacus sed turpis tincidunt id. Hendrerit dolor magna eget est lorem ipsum. Enim ut sem viverra aliquet. Eget nulla facilisi etiam dignissim diam quis enim lobortis scelerisque. Ac tortor dignissim convallis aenean et tortor at. Non tellus orci ac auctor augue. Nec dui nunc mattis enim ut tellus. Eget nunc lobortis mattis aliquam faucibus purus in massa tempor. Elementum nibh tellus molestie nunc. Ornare lectus sit amet est placerat in. Nec feugiat in fermentum posuere urna nec tincidunt praesent. Vestibulum rhoncus est pellentesque elit. Mollis nunc sed id semper risus in. Vitae elementum curabitur vitae nunc sed velit. Duis tristique sollicitudin nibh sit amet commodo nulla facilisi."""
+    )
+    return "Goodbye!"

test/example-has-stdout-and-tasks/example_has_stdout_and_tasks_test.py

@@ -0,0 +1,35 @@
+import unittest
+import pytest
+
+
+from example_has_stdout_and_tasks import hello, must_truncate
+
+
+class ExampleHasStdoutTest(unittest.TestCase):
+
+    @pytest.mark.task(taskno=1)
+    def test_hello(self):
+        self.assertEqual(hello(), "Hello, World!")
+
+    @pytest.mark.task(taskno=2)
+    def test_abc(self):
+        self.assertEqual(hello(), "Hello, World!")
+
+    @pytest.mark.task(taskno=3)
+    def test_trancation(self):
+        self.assertEqual(must_truncate(), "Hello, World!")
+
+
+class ExampleHasStdoutOtherTest(unittest.TestCase):
+
+    @pytest.mark.task(taskno=4)
+    def test_dummy(self):
+        self.assertEqual(hello(), "Hello, World!")
+
+    @pytest.mark.task(taskno=5)
+    def test_hello(self):
+        self.assertEqual(hello(), "Hello, World!")
+
+
+if __name__ == "__main__":
+    unittest.main()

test/example-has-stdout-and-tasks/results.json

@@ -0,0 +1,46 @@
+{
+  "version": 3,
+  "status": "fail",
+  "tests": [
+    {
+      "name": "ExampleHasStdoutTest.test_abc",
+      "status": "fail",
+      "message": "AssertionError: None != 'Hello, World!'",
+      "test_code": "def test_abc(self):\n    self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 2,
+      "output": "Hello, World!"
+    },
+    {
+      "name": "ExampleHasStdoutTest.test_hello",
+      "status": "fail",
+      "message": "AssertionError: None != 'Hello, World!'",
+      "test_code": "def test_hello(self):\n    self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 1,
+      "output": "Hello, World!"
+    },
+    {
+      "name": "ExampleHasStdoutTest.test_trancation",
+      "status": "fail",
+      "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!",
+      "test_code": "def test_trancation(self):\n    self.assertEqual(must_truncate(), \"Hello, World!\")",
+      "task_id": 3,
+      "output": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate s [Output was truncated. Please limit to 500 chars]"
+    },
+    {
+      "name": "ExampleHasStdoutOtherTest.test_dummy",
+      "status": "fail",
+      "message": "AssertionError: None != 'Hello, World!'",
+      "test_code": "def test_dummy(self):\n    self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 4,
+      "output": "Hello, World!"
+    },
+    {
+      "name": "ExampleHasStdoutOtherTest.test_hello",
+      "status": "fail",
+      "message": "AssertionError: None != 'Hello, World!'",
+      "test_code": "def test_hello(self):\n    self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 5,
+      "output": "Hello, World!"
+    }
+  ]
+}

test/example-has-stdout/results.json

@@ -1,40 +1,45 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "fail",
   "tests": [
     {
       "name": "ExampleHasStdoutTest.test_hello",
       "status": "fail",
       "message": "AssertionError: None != 'Hello, World!'",
       "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 0,
       "output": "Hello, World!"
     },
     {
       "name": "ExampleHasStdoutTest.test_abc",
       "status": "fail",
       "message": "AssertionError: None != 'Hello, World!'",
       "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 0,
       "output": "Hello, World!"
     },
     {
       "name": "ExampleHasStdoutTest.test_trancation",
       "status": "fail",
       "message": "AssertionError: 'Goodbye!' != 'Hello, World!'\n- Goodbye!\n+ Hello, World!",
       "test_code": "self.assertEqual(must_truncate(), \"Hello, World!\")",
+      "task_id": 0,
       "output": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Vulputate ut pharetra sit amet aliquam. Amet dictum sit amet justo donec enim diam vulputate ut. Consequat nisl vel pretium lectus quam id leo. Maecenas accumsan lacus vel facilisis volutpat est velit egestas dui. Faucibus et molestie ac feugiat sed. Fringilla phasellus faucibus scelerisque eleifend donec pretium vulputate s [Output was truncated. Please limit to 500 chars]"
     },
     {
       "name": "ExampleHasStdoutOtherTest.test_dummy",
       "status": "fail",
       "message": "AssertionError: None != 'Hello, World!'",
       "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 0,
       "output": "Hello, World!"
     },
     {
       "name": "ExampleHasStdoutOtherTest.test_hello",
       "status": "fail",
       "message": "AssertionError: None != 'Hello, World!'",
       "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 0,
       "output": "Hello, World!"
     }
   ]

test/example-partial-fail/results.json

@@ -1,28 +1,32 @@
 {
-  "version": 2,
+  "version": 3,
   "status": "fail",
   "tests": [
     {
       "name": "ExamplePartialFailTest.test_hello",
       "status": "fail",
       "message": "AssertionError: 'Hello, World!' != 'Goodbye'\n- Hello, World!\n+ Goodbye",
-      "test_code": "self.assertEqual(hello(), \"Goodbye\")"
+      "test_code": "self.assertEqual(hello(), \"Goodbye\")",
+      "task_id": 0
     },
     {
       "name": "ExamplePartialFailTest.test_abc",
       "status": "pass",
-      "test_code": "self.assertEqual(hello(), \"Hello, World!\")"
+      "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 0
     },
     {
       "name": "ExamplePartialFailOtherTest.test_dummy",
       "status": "fail",
       "message": "AssertionError: 'Hello, World!' != 'Goodbye'\n- Hello, World!\n+ Goodbye",
-      "test_code": "self.assertEqual(hello(), \"Goodbye\")"
+      "test_code": "self.assertEqual(hello(), \"Goodbye\")",
+      "task_id": 0
     },
     {
       "name": "ExamplePartialFailOtherTest.test_hello",
       "status": "pass",
-      "test_code": "self.assertEqual(hello(), \"Hello, World!\")"
+      "test_code": "self.assertEqual(hello(), \"Hello, World!\")",
+      "task_id": 0
     }
   ]
 }

test/example-partial-failure-with-subtests/example_partial_failure_with_subtests.py

@@ -0,0 +1,9 @@
+"""Example Exercism/Python solution file"""
+
+
+def hello(param):
+    if isinstance(param, int):
+        return ("Hello, World!")
+    else:
+        return ("Hello, World!", param)
+