From 1d58b6d02079d70e7962d75a7e98f6a8b24d7470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Tue, 1 Oct 2024 09:23:51 +0200 Subject: [PATCH] Store the original test, check and subresult outcome in results (#3147) --- docs/releases.rst | 7 +++++++ spec/plans/results.fmf | 38 +++++++++++++++++++++++++++++++++++--- tmt/result.py | 8 ++++++++ tmt/schemas/results.yaml | 9 +++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/docs/releases.rst b/docs/releases.rst index 8765a9a657..72c77295dc 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -40,6 +40,13 @@ reference, either branch, tag, git-describe, or if all fails the commit hash. You may encounter this in the verbose log of ``tmt tests show`` or plan/test imports. +:ref:`Result specification` now defines +``original-result`` key holding the original outcome of a test, subtest +or test checks. The effective outcome, stored in ``result`` key, is +computed from the original outcome, and it is affected by inputs like +:ref:`test result interpretation` or +:ref:`test checks`. + In verbose mode, the ``discover`` step now prints information about the beakerlib libraries which were fetched for the test execution. Use ``tmt run discover -vvv`` to see the details. diff --git a/spec/plans/results.fmf b/spec/plans/results.fmf index 9d2616dbc0..732ae155c6 100644 --- a/spec/plans/results.fmf +++ b/spec/plans/results.fmf @@ -27,9 +27,12 @@ description: | name: /test/one path: / - # String, outcome of the test execution. + # String, the effective outcome of the test execution. result: "pass"|"fail"|"info"|"warn"|"error"|"skip" + # String, the original outcome of the test execution. + original-result: "pass"|"fail"|"info"|"warn"|"error"|"skip" + # String, optional comment to report with the result. note: "Things were great." @@ -79,9 +82,12 @@ description: | # key. Fields have the same meaning as fields of the "parent" test result, but # relate to each check alone. check: - # String, outcome of the check execution. + # String, the effective outcome of the check execution. - result: "pass"|"fail"|"info"|"warn"|"error"|"skip" + # String, the original outcome of the check execution. + original-result: "pass"|"fail"|"info"|"warn"|"error"|"skip" + # String, optional comment to report with the result. note: "Things were great." @@ -114,9 +120,12 @@ description: | # String, name of the test phase. - name: First test case - # String, outcome of the phase execution. + # String, the effective outcome of the phase execution. result: "pass"|"fail"|"info"|"warn"|"error"|"skip" + # String, the original outcome of the phase execution. + original-result: "pass"|"fail"|"info"|"warn"|"error"|"skip" + # String, optional comment to report with the result. note: "Things were great." @@ -210,6 +219,26 @@ description: | original test result in the custom result set to the value known to tmt. + The ``original-result`` key holds the outcome of a test, check or + subresult as reported by the test, check or subresult itself, without + any additional influence. + A reported outcome may be a subject to interpretation, and tmt may + consider additional inputs and eventually report different effective + outcome in the ``result`` key. + + The following rules apply when it comes to test, test check and subresult + outcomes and their interpretation and effects: + + * Test outcome is interpreted according to + :py:ref:`/spec/tests/result`. It is not yet planned for test check + outcomes be interpreted. It is not yet planned for subresult + outcomes to be interpreted. + * Test checks and subresults do not influence the effective test + outcome. It is however planned for test check outcomes to affect + the test outcome, see https://github.com/teemtee/tmt/issues/3185 + for more details. It is not planned for subresults to affect + the test outcome, and we do not expect them to gain this effect. + See also the complete `JSON schema`__. For custom results files in JSON format, the same rules and schema @@ -224,6 +253,9 @@ description: | .. versionchanged:: 1.36 phase results are now renamed to the ``subresult`` key. + .. versionchanged:: 1.37 + original result of test, subtest and check is stored in ``original-result`` key. + __ https://github.com/teemtee/tmt/blob/main/tmt/schemas/results.yaml example: diff --git a/tmt/result.py b/tmt/result.py index 277300bc8b..62037791f1 100644 --- a/tmt/result.py +++ b/tmt/result.py @@ -123,6 +123,11 @@ class BaseResult(SerializableContainer): serialize=lambda result: result.value, unserialize=ResultOutcome.from_spec ) + original_result: ResultOutcome = field( + default=ResultOutcome.PASS, + serialize=lambda result: result.value, + unserialize=ResultOutcome.from_spec + ) note: Optional[str] = None log: list[Path] = field( default_factory=cast(Callable[[], list[Path]], list), @@ -133,6 +138,9 @@ class BaseResult(SerializableContainer): end_time: Optional[str] = None duration: Optional[str] = None + def __post_init__(self) -> None: + self.original_result = self.result + def show(self) -> str: """ Return a nicely colored result with test name (and note) """ diff --git a/tmt/schemas/results.yaml b/tmt/schemas/results.yaml index 6668c51428..8ce1ca6b0c 100644 --- a/tmt/schemas/results.yaml +++ b/tmt/schemas/results.yaml @@ -29,6 +29,9 @@ definitions: result: $ref: "/schemas/common#/definitions/result_outcome" + original-result: + $ref: "/schemas/common#/definitions/result_outcome" + note: $ref: "/schemas/common#/definitions/result_note" @@ -63,6 +66,9 @@ definitions: result: $ref: "/schemas/common#/definitions/result_outcome" + original-result: + $ref: "/schemas/common#/definitions/result_outcome" + note: $ref: "/schemas/common#/definitions/result_note" @@ -120,6 +126,9 @@ items: result: $ref: "/schemas/common#/definitions/result_outcome" + original-result: + $ref: "/schemas/common#/definitions/result_outcome" + context: $ref: "/schemas/common#/definitions/context"