feat: add process-test-results command (#424)

* feat: add process-test-results command

    this command locally processes test result files and makes a call to
    the GH API to create a comment

    This command should be run in Github Actions, it expects the
    provider-token option to contain the contents of the GITHUB_TOKEN
    env var.
* deps: update requirements.txt
* fix: use typing List instead of list

Signed-off-by: joseph-sentry <[email protected]>

.github/workflows/ci.yml

@@ -132,3 +132,38 @@ jobs:
       - name: Upload smart-labels
         run: |
           codecovcli --codecov-yml-path=codecov.yml do-upload --plugin pycoverage --plugin compress-pycoverage --fail-on-error -t ${{ secrets.CODECOV_TOKEN }} --flag smart-labels
+
+  test-process-test-results-cmd:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - python-version: "3.11"
+          - python-version: "3.10"
+          - python-version: "3.9"
+          - python-version: "3.8"
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+        fetch-depth: 2
+    - name: Set up Python ${{matrix.python-version}}
+      uses: actions/setup-python@v3
+      with:
+        python-version: "${{matrix.python-version}}"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        python setup.py develop
+        pip install -r tests/requirements.txt
+    - name: Test with pytest
+      run: |
+        pytest --cov --junitxml=junit.xml
+    - name: Dogfooding codecov-cli
+      if: ${{ !cancelled() }}
+      run: |
+        codecovcli process-test-results --provider-token ${{ secrets.GITHUB_TOKEN }}

codecov_cli/commands/process_test_results.py

@@ -0,0 +1,175 @@
+import logging
+import os
+import pathlib
+from dataclasses import dataclass
+from typing import List
+
+import click
+from test_results_parser import (
+    Outcome,
+    ParserError,
+    Testrun,
+    build_message,
+    parse_junit_xml,
+)
+
+from codecov_cli.helpers.request import (
+    log_warnings_and_errors_if_any,
+    send_post_request,
+)
+from codecov_cli.services.upload.file_finder import select_file_finder
+
+logger = logging.getLogger("codecovcli")
+
+
+_process_test_results_options = [
+    click.option(
+        "-s",
+        "--dir",
+        "--files-search-root-folder",
+        "dir",
+        help="Folder where to search for test results files",
+        type=click.Path(path_type=pathlib.Path),
+        default=pathlib.Path.cwd,
+        show_default="Current Working Directory",
+    ),
+    click.option(
+        "-f",
+        "--file",
+        "--files-search-direct-file",
+        "files",
+        help="Explicit files to upload. These will be added to the test results files to be processed. If you wish to only process the specified files, please consider using --disable-search to disable processing other files.",
+        type=click.Path(path_type=pathlib.Path),
+        multiple=True,
+        default=[],
+    ),
+    click.option(
+        "--exclude",
+        "--files-search-exclude-folder",
+        "exclude_folders",
+        help="Folders to exclude from search",
+        type=click.Path(path_type=pathlib.Path),
+        multiple=True,
+        default=[],
+    ),
+    click.option(
+        "--disable-search",
+        help="Disable search for coverage files. This is helpful when specifying what files you want to upload with the --file option.",
+        is_flag=True,
+        default=False,
+    ),
+    click.option(
+        "--provider-token",
+        help="Token used to make calls to Repo provider API",
+        type=str,
+        default=None,
+    ),
+]
+
+
+def process_test_results_options(func):
+    for option in reversed(_process_test_results_options):
+        func = option(func)
+    return func
+
+
+@dataclass
+class TestResultsNotificationPayload:
+    failures: List[Testrun]
+    failed: int = 0
+    passed: int = 0
+    skipped: int = 0
+
+
+@click.command()
+@process_test_results_options
+def process_test_results(
+    dir=None, files=None, exclude_folders=None, disable_search=None, provider_token=None
+):
+    if provider_token is None:
+        raise click.ClickException(
+            "Provider token was not provided. Make sure to pass --provider-token option with the contents of the GITHUB_TOKEN secret, so we can make a comment."
+        )
+
+    summary_file_path = os.getenv("GITHUB_STEP_SUMMARY")
+    if summary_file_path is None:
+        raise click.ClickException(
+            "Error getting step summary file path from environment. Can't find GITHUB_STEP_SUMMARY environment variable."
+        )
+
+    slug = os.getenv("GITHUB_REPOSITORY")
+    if slug is None:
+        raise click.ClickException(
+            "Error getting repo slug from environment. Can't find GITHUB_REPOSITORY environment variable."
+        )
+
+    ref = os.getenv("GITHUB_REF")
+    if ref is None or "pull" not in ref:
+        raise click.ClickException(
+            "Error getting PR number from environment. Can't find GITHUB_REF environment variable."
+        )
+
+    file_finder = select_file_finder(
+        dir, exclude_folders, files, disable_search, report_type="test_results"
+    )
+
+    upload_collection_results = file_finder.find_files()
+    if len(upload_collection_results) == 0:
+        raise click.ClickException(
+            "No JUnit XML files were found. Make sure to specify them using the --file option."
+        )
+
+    payload = generate_message_payload(upload_collection_results)
+
+    message = build_message(payload)
+
+    # write to step summary file
+    with open(summary_file_path, "w") as f:
+        f.write(message)
+
+    # GITHUB_REF is documented here: https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
+    pr_number = ref.split("/")[2]
+
+    create_github_comment(provider_token, slug, pr_number, message)
+
+
+def create_github_comment(token, repo_slug, pr_number, message):
+    url = f"https://api.github.com/repos/{repo_slug}/issues/{pr_number}/comments"
+
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "Authorization": f"Bearer {token}",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    logger.info("Posting github comment")
+
+    log_warnings_and_errors_if_any(
+        send_post_request(url=url, data={"body": message}, headers=headers),
+        "Posting test results comment",
+    )
+
+
+def generate_message_payload(upload_collection_results):
+    payload = TestResultsNotificationPayload(failures=[])
+
+    for result in upload_collection_results:
+        testruns = []
+        try:
+            logger.info(f"Parsing {result.get_filename()}")
+            testruns = parse_junit_xml(result.get_content())
+            for testrun in testruns:
+                if (
+                    testrun.outcome == Outcome.Failure
+                    or testrun.outcome == Outcome.Error
+                ):
+                    payload.failed += 1
+                    payload.failures.append(testrun)
+                elif testrun.outcome == Outcome.Skip:
+                    payload.skipped += 1
+                else:
+                    payload.passed += 1
+        except ParserError as err:
+            raise click.ClickException(
+                f"Error parsing {str(result.get_filename(), 'utf8')} with error: {err}"
+            )
+    return payload

codecov_cli/main.py

@@ -11,6 +11,7 @@
 from codecov_cli.commands.empty_upload import empty_upload
 from codecov_cli.commands.get_report_results import get_report_results
 from codecov_cli.commands.labelanalysis import label_analysis
+from codecov_cli.commands.process_test_results import process_test_results
 from codecov_cli.commands.report import create_report
 from codecov_cli.commands.send_notifications import send_notifications
 from codecov_cli.commands.staticanalysis import static_analysis
@@ -71,6 +72,7 @@ def cli(
 cli.add_command(empty_upload)
 cli.add_command(upload_process)
 cli.add_command(send_notifications)
+cli.add_command(process_test_results)
 
 
 def run():

requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile setup.py
@@ -15,8 +15,6 @@ charset-normalizer==3.3.0
     # via requests
 click==8.1.7
     # via codecov-cli (setup.py)
-exceptiongroup==1.1.3
-    # via anyio
 h11==0.14.0
     # via httpcore
 httpcore==0.16.3
@@ -32,19 +30,21 @@ ijson==3.2.3
     # via codecov-cli (setup.py)
 pyyaml==6.0.1
     # via codecov-cli (setup.py)
+regex==2023.12.25
+    # via codecov-cli (setup.py)
 requests==2.31.0
     # via responses
 responses==0.21.0
     # via codecov-cli (setup.py)
 rfc3986[idna2008]==1.5.0
-    # via
-    #   httpx
-    #   rfc3986
+    # via httpx
 sniffio==1.3.0
     # via
     #   anyio
     #   httpcore
     #   httpx
+test-results-parser==0.1.0
+    # via codecov-cli (setup.py)
 tree-sitter==0.20.2
     # via codecov-cli (setup.py)
 urllib3==2.0.6

samples/junit.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<testsuites>
+    <testsuite name="pytest" errors="0" failures="1" skipped="0" tests="4" time="0.052"
+        timestamp="2023-11-06T11:17:04.011072" hostname="VFHNWJDWH9.local">
+        <testcase classname="api.temp.calculator.test_calculator" name="test_add" time="0.001" />
+        <testcase classname="api.temp.calculator.test_calculator" name="test_subtract" time="0.001" />
+        <testcase classname="api.temp.calculator.test_calculator" name="test_multiply" time="0.000" />
+        <testcase classname="api.temp.calculator.test_calculator" name="test_divide" time="0.001">
+            <failure
+                message="assert 1.0 == 0.5&#10; +  where 1.0 = &lt;function Calculator.divide at 0x104c9eb90&gt;(1, 2)&#10; +    where &lt;function Calculator.divide at 0x104c9eb90&gt; = Calculator.divide">def
+                test_divide():
+                &gt; assert Calculator.divide(1, 2) == 0.5
+                E assert 1.0 == 0.5
+                E + where 1.0 = &lt;function Calculator.divide at 0x104c9eb90&gt;(1, 2)
+                E + where &lt;function Calculator.divide at 0x104c9eb90&gt; = Calculator.divide
+                api/temp/calculator/test_calculator.py:30: AssertionError</failure>
+        </testcase>
+    </testsuite>
+</testsuites>

setup.py

@@ -24,6 +24,8 @@
         "pyyaml==6.*",
         "responses==0.21.*",
         "tree-sitter==0.20.*",
+        "test-results-parser==0.1.*",
+        "regex",
     ],
     entry_points={
         "console_scripts": [