From b626b7dcde69c9f214281a65992c1a873c46cd65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Wed, 18 Sep 2024 11:08:28 +0200 Subject: [PATCH 1/3] Provide mapping between results and tests for easier reporting --- tmt/steps/execute/__init__.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tmt/steps/execute/__init__.py b/tmt/steps/execute/__init__.py index 2e54718c02..f2a8945fb2 100644 --- a/tmt/steps/execute/__init__.py +++ b/tmt/steps/execute/__init__.py @@ -1169,3 +1169,29 @@ def results(self) -> list["tmt.result.Result"]: https://tmt.readthedocs.io/en/latest/spec/plans.html#execute """ return self._results + + def results_for_tests(self, tests: list['tmt.base.Test']) \ + -> list[tuple[Optional[Result], Optional['tmt.base.Test']]]: + """ + Collect results and corresponding tests. + + :returns: a list of result and test pairs. + * if there is not test found for the result, e.g. when + results were loaded from storage but tests were not, + ``None`` represents the missing test: ``(result, None)``. + * if there is no result for a test, e.g. when the test was + not executed, ``None`` represents the missing result: + ``(None, test)``. + """ + + known_serial_numbers = {test.serial_number: test for test in tests} + referenced_serial_numbers = {result.serial_number for result in self._results} + + return [ + (result, known_serial_numbers.get(result.serial_number)) + for result in self._results + ] + [ + (None, test) + for test in tests + if test.serial_number not in referenced_serial_numbers + ] From f59bc68f8c5a42879df0ae98c12031c37e197c67 Mon Sep 17 00:00:00 2001 From: Karel Srot Date: Wed, 18 Sep 2024 10:20:56 +0200 Subject: [PATCH 2/3] Test results reporting rewrite Modifies test results reporting in the following way: - All results from execute.results() are reported. - All tests from discover.tests() that have not been executed are reported as 'skipped' at the end. --- tmt/steps/report/reportportal.py | 199 +++++++++++++++---------------- 1 file changed, 99 insertions(+), 100 deletions(-) diff --git a/tmt/steps/report/reportportal.py b/tmt/steps/report/reportportal.py index fadff6cdb0..91d0c2f5d6 100644 --- a/tmt/steps/report/reportportal.py +++ b/tmt/steps/report/reportportal.py @@ -8,7 +8,7 @@ import tmt.log import tmt.steps.report -from tmt.result import Result, ResultOutcome +from tmt.result import ResultOutcome from tmt.utils import field, yaml_to_dict if TYPE_CHECKING: @@ -511,125 +511,124 @@ def go(self, *, logger: Optional[tmt.log.Logger] = None) -> None: self.verbose("uuid", suite_uuid, "yellow", shift=1) self.data.suite_uuid = suite_uuid - # For each test - for test in self.step.plan.discover.tests(): - + for result, test in self.step.plan.execute.results_for_tests( + self.step.plan.discover.tests()): test_time = self.time() - - results: list[Optional[Result]] = [None] - if executed: - results = [result for result in self.step.plan.execute.results() - if test.serial_number == result.serial_number] - - # process even tests not having test results - for result in results: - - # TODO: for happz, connect Test to Result if possible - # (but now it is probably too hackish to be fixed) - - item_attributes = attributes.copy() - if result: - test_time = result.start_time or self.time() - - # for guests, save their primary address - if result.guest.primary_address: - item_attributes.append({ - 'key': 'guest_primary_address', - 'value': result.guest.primary_address}) - - # for multi-host tests store also guest name and role - if result.guest.name != 'default-0': - item_attributes.append( - {'key': 'guest_name', 'value': result.guest.name}) - if result.guest.role: - item_attributes.append( - {'key': 'guest_role', 'value': result.guest.role}) - + test_name = None + test_description = '' + test_link = None + test_id = None + env_vars = None + + item_attributes = attributes.copy() + if result: + serial_number = result.serial_number + test_name = result.name + test_time = result.start_time or self.time() + # for guests, save their primary address + if result.guest.primary_address: + item_attributes.append({ + 'key': 'guest_primary_address', + 'value': result.guest.primary_address}) + # for multi-host tests store also provision name and role + if result.guest.name != 'default-0': + item_attributes.append( + {'key': 'guest_name', 'value': result.guest.name}) + if result.guest.role: + item_attributes.append( + {'key': 'guest_role', 'value': result.guest.role}) + + # update RP item with additional attributes if test details are available + if test: + serial_number = test.serial_number + if not test_name: + test_name = test.name if test.contact: item_attributes.append({"key": "contact", "value": test.contact[0]}) + if test.summary: + test_description = test.summary + if test.web_link(): + test_link = test.web_link() + if test.id: + test_id = test.id env_vars = [ {'key': key, 'value': value} for key, value in test.environment.items() if not re.search(envar_pattern, key)] - if create_test: - - test_description = test.summary or "" - if ((self.data.upload_to_launch and launch_per_plan) - or self.data.upload_to_suite): - test_description = self.append_description(test_description) + if create_test: + if ((self.data.upload_to_launch and launch_per_plan) + or self.data.upload_to_suite): + test_description = self.append_description(test_description) - # Create a test item - self.info("test", test.name, color="cyan") + # Create a test item + self.info("test", test_name, color="cyan") + response = self.rp_api_post( + session=session, + path=f"item{f'/{suite_uuid}' if suite_uuid else ''}", + json={"name": test_name, + "description": test_description, + "attributes": item_attributes, + "parameters": env_vars, + "codeRef": test_link, + "launchUuid": launch_uuid, + "type": "step", + "testCaseId": test_id, + "startTime": test_time}) + item_uuid = yaml_to_dict(response.text).get("id") + assert item_uuid is not None + self.verbose("uuid", item_uuid, "yellow", shift=1) + self.data.test_uuids[serial_number] = item_uuid + else: + item_uuid = self.data.test_uuids[serial_number] + # Support for idle tests + status = "SKIPPED" + if result: + # For each log + for index, log_path in enumerate(result.log): + try: + log = self.step.plan.execute.read(log_path) + except tmt.utils.FileError: + continue + + level = "INFO" if log_path == result.log[0] else "TRACE" + status = self.TMT_TO_RP_RESULT_STATUS[result.result] + + # Upload log response = self.rp_api_post( session=session, - path=f"item{f'/{suite_uuid}' if suite_uuid else ''}", - json={"name": test.name, - "description": test_description, - "attributes": item_attributes, - "parameters": env_vars, - "codeRef": test.web_link() or None, + path="log/entry", + json={"message": _filter_invalid_chars(log), + "itemUuid": item_uuid, "launchUuid": launch_uuid, - "type": "step", - "testCaseId": test.id or None, - "startTime": test_time}) - item_uuid = yaml_to_dict(response.text).get("id") - assert item_uuid is not None - self.verbose("uuid", item_uuid, "yellow", shift=1) - self.data.test_uuids[test.serial_number] = item_uuid - else: - item_uuid = self.data.test_uuids[test.serial_number] - - # Support for idle tests - status = "SKIPPED" - if executed and result: - # For each log - for index, log_path in enumerate(result.log): - try: - log = self.step.plan.execute.read(log_path) - except tmt.utils.FileError: - continue - - level = "INFO" if log_path == result.log[0] else "TRACE" - status = self.TMT_TO_RP_RESULT_STATUS[result.result] - - # Upload log + "level": level, + "time": result.end_time}) + + # Write out failures + if index == 0 and status == "FAILED": + message = _filter_invalid_chars(result.failures(log)) response = self.rp_api_post( session=session, path="log/entry", - json={"message": _filter_invalid_chars(log), + json={"message": message, "itemUuid": item_uuid, "launchUuid": launch_uuid, - "level": level, + "level": "ERROR", "time": result.end_time}) - # Write out failures - if index == 0 and status == "FAILED": - message = _filter_invalid_chars(result.failures(log)) - response = self.rp_api_post( - session=session, - path="log/entry", - json={"message": message, - "itemUuid": item_uuid, - "launchUuid": launch_uuid, - "level": "ERROR", - "time": result.end_time}) - - # TODO: Add tmt files as attachments + test_time = result.end_time or self.time() - test_time = result.end_time or self.time() - - # Finish the test item - response = self.rp_api_put( - session=session, - path=f"item/{item_uuid}", - json={ - "launchUuid": launch_uuid, - "endTime": test_time, - "status": status, - "issue": { - "issueType": self.get_defect_type_locator(session, defect_type)}}) - launch_time = test_time + # Finish the test item + response = self.rp_api_put( + session=session, + path=f"item/{item_uuid}", + json={ + "launchUuid": launch_uuid, + "endTime": test_time, + "status": status, + "issue": { + "issueType": self.get_defect_type_locator(session, defect_type)}}) + launch_time = test_time if create_suite: # Finish the test suite From 2a030dd908b133e80c9684b07af20b594137e74c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pl=C3=ADchal?= Date: Tue, 1 Oct 2024 21:23:29 +0200 Subject: [PATCH 3/3] Add a short release note Plus a minor style adjustment. --- docs/releases.rst | 4 ++++ tmt/steps/execute/__init__.py | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/releases.rst b/docs/releases.rst index 7861e8c8fb..0b62eec8f3 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -66,6 +66,10 @@ The :ref:`/plugins/provision/beaker` provision plugin now newly supports providing a custom :ref:`/spec/plans/provision/kickstart` configuration. +The :ref:`/plugins/report/reportportal` plugin now uploads the +complete set of discovered tests, including those which have not +been executed. These tests are marked as ``skipped``. + tmt-1.36.1 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tmt/steps/execute/__init__.py b/tmt/steps/execute/__init__.py index f2a8945fb2..e007bf331a 100644 --- a/tmt/steps/execute/__init__.py +++ b/tmt/steps/execute/__init__.py @@ -1170,8 +1170,10 @@ def results(self) -> list["tmt.result.Result"]: """ return self._results - def results_for_tests(self, tests: list['tmt.base.Test']) \ - -> list[tuple[Optional[Result], Optional['tmt.base.Test']]]: + def results_for_tests( + self, + tests: list['tmt.base.Test'] + ) -> list[tuple[Optional[Result], Optional['tmt.base.Test']]]: """ Collect results and corresponding tests.